#13159

pending completion

Build # #13159

Build Type

push

circleci

Committed by

Vinnl

Commit Message

Merge branch 'main' into MNTOR-1309-exposure-scan

Run Details

282 of 1655 branches covered (17.04%)

Branch coverage included in aggregate %.

10 of 10 new or added lines in 1 file covered. (100.0%)

959 of 4464 relevant lines covered (21.48%)

1.79 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/src/scripts/migrationCleanup.js

/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

/**
 * Executes once
 * The purpose of the script is to clean up some of the failed records during db migration on 3/28/23
 */

import Knex from 'knex'
import knexConfig from '../db/knexfile.js'
import { getAllBreachesFromDb } from '../utils/hibp.js'
import { getAllEmailsAndBreaches } from '../utils/breaches.js'
import { setBreachResolution } from '../db/tables/subscribers.js'
import mozlog from '../utils/log.js'
const log = mozlog('script.migrationCleanup')
const knex = Knex(knexConfig)

const LIMIT = 3000
let subscribersArr = []
// load all breaches for ref
const allBreaches = await getAllBreachesFromDb()
if (allBreaches && allBreaches.length > 0) log.info('breach_count', 'breaches loaded successfully! ', allBreaches.length)

const count = await knex
  .from('subscribers')
  .whereRaw('NOT ((breach_resolution)::jsonb \\? \'useBreachId\')')
  .count('*')

log.info('total_to_be_executed', count[0])

// find all subscribers who resolved any breaches in the past,
// replace recency index with breach id

for (let i = 0; i < 10; i++) {
  subscribersArr = await knex
    .select('id', 'primary_email', 'breach_resolution')
    .from('subscribers')
    .orderBy('updated_at', 'desc')
    .whereRaw('NOT ((breach_resolution)::jsonb \\? \'useBreachId\')')
    .limit(LIMIT)

  log.info('job', `Loaded # of subscribers: ${subscribersArr.length}`)

  for (const subscriber of subscribersArr) {
    const { breach_resolution: v2 } = subscriber
    // console.debug({ v2 })

    // if useBreachId is set, skip because this breach_resolution has already been worked on
    if (v2.useBreachId) {
      log.warn('job', 'Skipping since `useBreachId` is set already, this breach resolution is already converted')
      continue
    }

    const newResolutions = {}

    // fetch subscriber all breaches / email
    const subscriberBreachesEmail = await getAllEmailsAndBreaches(subscriber, allBreaches)
    // console.debug(JSON.stringify(subscriberBreachesEmail.verifiedEmails))

    for (const email in v2) {
    // console.debug({ email })
      const resolutions = v2[email]
      // console.debug({ resolutions })
      newResolutions[email] = {}

      for (const recencyIndex in resolutions) {
        console.debug({ recencyIndex })

        // find subscriber's relevant recency index breach information
        const ve = subscriberBreachesEmail.verifiedEmails?.filter(ve => ve.email === email)[0] || {}
        const subBreach = ve.breaches?.filter(b => Number(b.recencyIndex) === Number(recencyIndex))[0] || null
        const breachName = subBreach?.Name
        console.debug({ breachName })

        // find breach id for the breach
        const breachId = allBreaches.find(b => b.Name === breachName)?.Id
        log.info('job', { breachId })
        newResolutions[email][breachId] = v2[email][recencyIndex]
      }
    }

    // check if v2 is changed, if so, upsert the new v2
    newResolutions.useBreachId = true
    await setBreachResolution(subscriber, newResolutions)
  }
}

// breaking out of do..while loop
log.info('job', 'Reaching the end of the table')
process.exit()

1	/* This Source Code Form is subject to the terms of the Mozilla Public
2	* License, v. 2.0. If a copy of the MPL was not distributed with this
3	* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
4
5	/**
6	* Executes once
7	* The purpose of the script is to clean up some of the failed records during db migration on 3/28/23
8	*/
9
10	import Knex from 'knex'
11	import knexConfig from '../db/knexfile.js'
12	import { getAllBreachesFromDb } from '../utils/hibp.js'
13	import { getAllEmailsAndBreaches } from '../utils/breaches.js'
14	import { setBreachResolution } from '../db/tables/subscribers.js'
15	import mozlog from '../utils/log.js'
16	const log = mozlog('script.migrationCleanup')	×
17	const knex = Knex(knexConfig)	×
18
19	const LIMIT = 3000	×
20	let subscribersArr = []	×
21	// load all breaches for ref
22	const allBreaches = await getAllBreachesFromDb()	×
23	if (allBreaches && allBreaches.length > 0) log.info('breach_count', 'breaches loaded successfully! ', allBreaches.length)	×
24
25	const count = await knex	×
26	.from('subscribers')
27	.whereRaw('NOT ((breach_resolution)::jsonb \\? \'useBreachId\')')
28	.count('*')
29
30	log.info('total_to_be_executed', count[0])	×
31
32	// find all subscribers who resolved any breaches in the past,
33	// replace recency index with breach id
34
35	for (let i = 0; i < 10; i++) {	×
36	subscribersArr = await knex	×
37	.select('id', 'primary_email', 'breach_resolution')
38	.from('subscribers')
39	.orderBy('updated_at', 'desc')
40	.whereRaw('NOT ((breach_resolution)::jsonb \\? \'useBreachId\')')
41	.limit(LIMIT)
42
43	log.info('job', `Loaded # of subscribers: ${subscribersArr.length}`)	×
44
45	for (const subscriber of subscribersArr) {	×
46	const { breach_resolution: v2 } = subscriber	×
47	// console.debug({ v2 })
48
49	// if useBreachId is set, skip because this breach_resolution has already been worked on
50	if (v2.useBreachId) {	×
51	log.warn('job', 'Skipping since `useBreachId` is set already, this breach resolution is already converted')	×
52	continue	×
53	}
54
55	const newResolutions = {}	×
56
57	// fetch subscriber all breaches / email
58	const subscriberBreachesEmail = await getAllEmailsAndBreaches(subscriber, allBreaches)	×
59	// console.debug(JSON.stringify(subscriberBreachesEmail.verifiedEmails))
60
61	for (const email in v2) {	×
62	// console.debug({ email })
63	const resolutions = v2[email]	×
64	// console.debug({ resolutions })
65	newResolutions[email] = {}	×
66
67	for (const recencyIndex in resolutions) {	×
68	console.debug({ recencyIndex })	×
69
70	// find subscriber's relevant recency index breach information
71	const ve = subscriberBreachesEmail.verifiedEmails?.filter(ve => ve.email === email)[0] \|\| {}	×
72	const subBreach = ve.breaches?.filter(b => Number(b.recencyIndex) === Number(recencyIndex))[0] \|\| null	×
73	const breachName = subBreach?.Name	×
74	console.debug({ breachName })	×
75
76	// find breach id for the breach
77	const breachId = allBreaches.find(b => b.Name === breachName)?.Id	×
78	log.info('job', { breachId })	×
79	newResolutions[email][breachId] = v2[email][recencyIndex]	×
80	}
81	}
82
83	// check if v2 is changed, if so, upsert the new v2
84	newResolutions.useBreachId = true	×
85	await setBreachResolution(subscriber, newResolutions)	×
86	}
87	}
88
89	// breaking out of do..while loop
90	log.info('job', 'Reaching the end of the table')	×
91	process.exit()	×

mozilla / blurts-server / #13159

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous