45ba77d7-fd3c-4064-90eb-157d6aa10611

Build # 45ba77d7-fd3c-4064-90eb-157d6aa10611

Build Type

push

circleci

Committed by Robert Helmer

Commit Message

update the queue adr

Run Details

282 of 1631 branches covered (17.29%)

Branch coverage included in aggregate %.

959 of 4375 relevant lines covered (21.92%)

3.65 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/src/scripts/recencyToBreachId.js

/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

/**
 * Executes once
 * Execute after `deprecateBreachResolve.js`
 * The purpose of the script is to convert all recency indices to breach ids for `breach_resolution`
 * For backwards compatibility, all converted `breach_resolution` fields will have a boolean
 * `useBreachId: true/false`
 */

import Knex from 'knex'
import knexConfig from '../db/knexfile.js'
import { getAllBreachesFromDb } from '../utils/hibp.js'
import { getAllEmailsAndBreaches } from '../utils/breaches.js'
import { setBreachResolution } from '../db/tables/subscribers.js'
const knex = Knex(knexConfig)

const LIMIT = 50 // with millions of records, we have to load a few at a time
let offset = 0 // looping through all records with offset
let subscribersArr = []

// load all breaches for ref
const allBreaches = await getAllBreachesFromDb()
if (allBreaches && allBreaches.length > 0) console.log('breaches loaded successfully! ', allBreaches.length)
console.log(JSON.stringify(allBreaches[0]))

// find all subscribers who resolved any breaches in the past,
// replace recency index with breach id
do {
  console.log(`Converting breach_resolution to use breach Id - start: ${offset} limit: ${LIMIT}`)
  subscribersArr = await knex
    .select('id', 'primary_email', 'breach_resolution')
    .from('subscribers')
    .whereNotNull('breach_resolution')
    .limit(LIMIT)
    .offset(offset)

  console.log(`Loaded # of subscribers: ${subscribersArr.length}`)

  for (const subscriber of subscribersArr) {
    const { breach_resolution: v2 } = subscriber
    console.debug({ v2 })

    // if useBreachId is set, skip because this breach_resolution has already been worked on
    if (v2.useBreachId) {
      console.log('Skipping since `useBreachId` is set already, this breach resolution is already converted')
      continue
    }

    const newResolutions = {}

    // fetch subscriber all breaches / email
    const subscriberBreachesEmail = await getAllEmailsAndBreaches(subscriber, allBreaches)
    console.debug(JSON.stringify(subscriberBreachesEmail.verifiedEmails))

    for (const email in v2) {
      console.debug({ email })
      const resolutions = v2[email]
      console.debug({ resolutions })
      newResolutions[email] = {}

      for (const recencyIndex in resolutions) {
        console.debug({ recencyIndex })

        // find subscriber's relevant recency index breach information
        const ve = subscriberBreachesEmail.verifiedEmails?.filter(ve => ve.email === email)[0] || {}
        const subBreach = ve.breaches?.filter(b => Number(b.recencyIndex) === Number(recencyIndex))[0] || null
        const breachName = subBreach?.Name
        console.debug({ breachName })

        // find breach id for the breach
        const breachId = allBreaches.find(b => b.Name === breachName)?.Id
        console.log({ breachId })
        newResolutions[email][breachId] = v2[email][recencyIndex]
      }
    }

    // check if v2 is changed, if so, upsert the new v2
    newResolutions.useBreachId = true
    console.log(JSON.stringify(newResolutions))
    await setBreachResolution(subscriber, newResolutions)
  }
  offset += LIMIT
} while (subscribersArr.length === LIMIT)

// breaking out of do..while loop
console.log('Reaching the end of the table, offset ended at', offset)
process.exit()

1	/* This Source Code Form is subject to the terms of the Mozilla Public
2	* License, v. 2.0. If a copy of the MPL was not distributed with this
3	* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
4
5	/**
6	* Executes once
7	* Execute after `deprecateBreachResolve.js`
8	* The purpose of the script is to convert all recency indices to breach ids for `breach_resolution`
9	* For backwards compatibility, all converted `breach_resolution` fields will have a boolean
10	* `useBreachId: true/false`
11	*/
12
13	import Knex from 'knex'
14	import knexConfig from '../db/knexfile.js'
15	import { getAllBreachesFromDb } from '../utils/hibp.js'
16	import { getAllEmailsAndBreaches } from '../utils/breaches.js'
17	import { setBreachResolution } from '../db/tables/subscribers.js'
18	const knex = Knex(knexConfig)	×
19
20	const LIMIT = 50 // with millions of records, we have to load a few at a time	×
21	let offset = 0 // looping through all records with offset	×
22	let subscribersArr = []	×
23
24	// load all breaches for ref
25	const allBreaches = await getAllBreachesFromDb()	×
26	if (allBreaches && allBreaches.length > 0) console.log('breaches loaded successfully! ', allBreaches.length)	×
27	console.log(JSON.stringify(allBreaches[0]))	×
28
29	// find all subscribers who resolved any breaches in the past,
30	// replace recency index with breach id
31	do {	×
32	console.log(`Converting breach_resolution to use breach Id - start: ${offset} limit: ${LIMIT}`)	×
33	subscribersArr = await knex	×
34	.select('id', 'primary_email', 'breach_resolution')
35	.from('subscribers')
36	.whereNotNull('breach_resolution')
37	.limit(LIMIT)
38	.offset(offset)
39
40	console.log(`Loaded # of subscribers: ${subscribersArr.length}`)	×
41
42	for (const subscriber of subscribersArr) {	×
43	const { breach_resolution: v2 } = subscriber	×
44	console.debug({ v2 })	×
45
46	// if useBreachId is set, skip because this breach_resolution has already been worked on
47	if (v2.useBreachId) {	×
48	console.log('Skipping since `useBreachId` is set already, this breach resolution is already converted')	×
49	continue	×
50	}
51
52	const newResolutions = {}	×
53
54	// fetch subscriber all breaches / email
55	const subscriberBreachesEmail = await getAllEmailsAndBreaches(subscriber, allBreaches)	×
56	console.debug(JSON.stringify(subscriberBreachesEmail.verifiedEmails))	×
57
58	for (const email in v2) {	×
59	console.debug({ email })	×
60	const resolutions = v2[email]	×
61	console.debug({ resolutions })	×
62	newResolutions[email] = {}	×
63
64	for (const recencyIndex in resolutions) {	×
65	console.debug({ recencyIndex })	×
66
67	// find subscriber's relevant recency index breach information
68	const ve = subscriberBreachesEmail.verifiedEmails?.filter(ve => ve.email === email)[0] \|\| {}	×
69	const subBreach = ve.breaches?.filter(b => Number(b.recencyIndex) === Number(recencyIndex))[0] \|\| null	×
70	const breachName = subBreach?.Name	×
71	console.debug({ breachName })	×
72
73	// find breach id for the breach
74	const breachId = allBreaches.find(b => b.Name === breachName)?.Id	×
75	console.log({ breachId })	×
76	newResolutions[email][breachId] = v2[email][recencyIndex]	×
77	}
78	}
79
80	// check if v2 is changed, if so, upsert the new v2
81	newResolutions.useBreachId = true	×
82	console.log(JSON.stringify(newResolutions))	×
83	await setBreachResolution(subscriber, newResolutions)	×
84	}
85	offset += LIMIT	×
86	} while (subscribersArr.length === LIMIT)
87
88	// breaking out of do..while loop
89	console.log('Reaching the end of the table, offset ended at', offset)	×
90	process.exit()	×

mozilla / blurts-server / 45ba77d7-fd3c-4064-90eb-157d6aa10611

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous