b86637bb-ffe8-4e59-ade8-c8caf0aa8efc

Build # b86637bb-ffe8-4e59-ade8-c8caf0aa8efc

Build Type

push

circleci

Committed by GitHub

Commit Message

Merge pull request #2757 from mozilla/MNTOR-977

Run Details

282 of 1160 branches covered (24.31%)

Branch coverage included in aggregate %.

75 of 75 new or added lines in 2 files covered. (100.0%)

959 of 3091 relevant lines covered (31.03%)

5.04 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/src/scripts/convertBreachResolutions.js

/**
 * Executes once
 * The purpose of the script is to convert all `subscriber.breaches_resolved` to `subscriber.breaches_resolution`
 * with the goal of deprecating the column
 */

import Knex from 'knex'
import knexConfig from '../db/knexfile.js'
import { getAllBreachesFromDb } from '../utils/hibp.js'
import { getAllEmailsAndBreaches } from '../utils/breaches.js'
import { setBreachResolution } from '../db/tables/subscribers.js'
import { BreachDataTypes } from '../utils/breach-resolution.js'
const knex = Knex(knexConfig)

const LIMIT = 50 // with millions of records, we have to load a few at a time
let offset = 0 // looping through all records with offset
let subscribersArr = []

// load all breaches for ref
const allBreaches = await getAllBreachesFromDb()
if (allBreaches && allBreaches.length > 0) console.log('breaches loaded successfully! ', allBreaches.length)

// find all subscribers who resolved any breaches in the past, convert those
// records into the new v2 format
do {
  console.log(`Converting breaches_resolved to breach_resolution - start: ${offset} limit: ${LIMIT}`)
  subscribersArr = await knex
    .select('id', 'primary_email', 'breaches_resolved', 'breach_resolution')
    .from('subscribers')
    .whereNotNull('breaches_resolved')
    .limit(LIMIT)
    .offset(offset)

  console.log(`Loaded # of subscribers: ${subscribersArr.length}`)

  for (const subscriber of subscribersArr) {
    let { breaches_resolved: v1, breach_resolution: v2 } = subscriber
    console.debug({ v1 })
    console.debug({ v2 })

    let isV2Changed = false // use a boolean to track if v2 has been changed, only upsert if so

    // fetch subscriber all breaches / email
    const subscriberBreachesEmail = await getAllEmailsAndBreaches(subscriber, allBreaches)
    console.debug(JSON.stringify(subscriberBreachesEmail.verifiedEmails))

    for (const [email, resolvedRecencyIndices] of Object.entries(v1)) {
      console.debug({ email })
      console.debug({ resolvedRecencyIndices })
      for (const recencyIndex of resolvedRecencyIndices) {
        console.debug({ recencyIndex })
        // find subscriber's relevant recency index breach information
        const ve = subscriberBreachesEmail.verifiedEmails?.filter(ve => ve.email === email)[0] || {}
        const subBreach = ve.breaches?.filter(b => Number(b.recencyIndex) === Number(recencyIndex))[0] || null
        console.debug({ subBreach })

        if (!subBreach || !subBreach.DataClasses) {
          console.warn(`SKIP: Cannot find subscribers breach and data types - recency: ${recencyIndex} email: ${email}`)
          continue
        }

        // if email does not exist in v2, we need to add it to the object
        // format: {email: { recencyIndex: { isResolved: true, resolutionsChecked: [DataTypes]}}}
        if (!v2) v2 = {}
        if (!v2[email]) {
          v2[email] = {
            [recencyIndex]: {
              isResolved: true,
              resolutionsChecked: subBreach?.DataClasses || [BreachDataTypes.General]
            }
          }

          isV2Changed = true
        }
        if (v2[email][recencyIndex]?.isResolved) {
          console.log(`recencyIndex ${recencyIndex} exists in v2 and is resolved, no changes`)
        } else {
          console.log(`recencyIndex ${recencyIndex} either does not exist or is not resolved, overwriting`)
          v2[email][recencyIndex] = {
            isResolved: true,
            resolutionsChecked: subBreach?.DataClasses
          }
          isV2Changed = true
        }
      }
    }

    // check if v2 is changed, if so, upsert the new v2
    if (isV2Changed) {
      await setBreachResolution(subscriber, v2)
    }
  }
  offset += LIMIT
} while (subscribersArr.length === LIMIT)

// breaking out of do..while loop
console.log('Reaching the end of the table, offset ended at', offset)
process.exit()

1	/**
2	* Executes once
3	* The purpose of the script is to convert all `subscriber.breaches_resolved` to `subscriber.breaches_resolution`
4	* with the goal of deprecating the column
5	*/
6
7	import Knex from 'knex'
8	import knexConfig from '../db/knexfile.js'
9	import { getAllBreachesFromDb } from '../utils/hibp.js'
10	import { getAllEmailsAndBreaches } from '../utils/breaches.js'
11	import { setBreachResolution } from '../db/tables/subscribers.js'
12	import { BreachDataTypes } from '../utils/breach-resolution.js'
13	const knex = Knex(knexConfig)	×
14
15	const LIMIT = 50 // with millions of records, we have to load a few at a time	×
16	let offset = 0 // looping through all records with offset	×
17	let subscribersArr = []	×
18
19	// load all breaches for ref
20	const allBreaches = await getAllBreachesFromDb()	×
21	if (allBreaches && allBreaches.length > 0) console.log('breaches loaded successfully! ', allBreaches.length)	×
22
23	// find all subscribers who resolved any breaches in the past, convert those
24	// records into the new v2 format
25	do {	×
26	console.log(`Converting breaches_resolved to breach_resolution - start: ${offset} limit: ${LIMIT}`)	×
27	subscribersArr = await knex	×
28	.select('id', 'primary_email', 'breaches_resolved', 'breach_resolution')
29	.from('subscribers')
30	.whereNotNull('breaches_resolved')
31	.limit(LIMIT)
32	.offset(offset)
33
34	console.log(`Loaded # of subscribers: ${subscribersArr.length}`)	×
35
36	for (const subscriber of subscribersArr) {	×
37	let { breaches_resolved: v1, breach_resolution: v2 } = subscriber	×
38	console.debug({ v1 })	×
39	console.debug({ v2 })	×
40
41	let isV2Changed = false // use a boolean to track if v2 has been changed, only upsert if so	×
42
43	// fetch subscriber all breaches / email
44	const subscriberBreachesEmail = await getAllEmailsAndBreaches(subscriber, allBreaches)	×
45	console.debug(JSON.stringify(subscriberBreachesEmail.verifiedEmails))	×
46
47	for (const [email, resolvedRecencyIndices] of Object.entries(v1)) {	×
48	console.debug({ email })	×
49	console.debug({ resolvedRecencyIndices })	×
50	for (const recencyIndex of resolvedRecencyIndices) {	×
51	console.debug({ recencyIndex })	×
52	// find subscriber's relevant recency index breach information
53	const ve = subscriberBreachesEmail.verifiedEmails?.filter(ve => ve.email === email)[0] \|\| {}	×
54	const subBreach = ve.breaches?.filter(b => Number(b.recencyIndex) === Number(recencyIndex))[0] \|\| null	×
55	console.debug({ subBreach })	×
56
57	if (!subBreach \|\| !subBreach.DataClasses) {	×
58	console.warn(`SKIP: Cannot find subscribers breach and data types - recency: ${recencyIndex} email: ${email}`)	×
59	continue	×
60	}
61
62	// if email does not exist in v2, we need to add it to the object
63	// format: {email: { recencyIndex: { isResolved: true, resolutionsChecked: [DataTypes]}}}
64	if (!v2) v2 = {}	×
65	if (!v2[email]) {	×
66	v2[email] = {	×
67	[recencyIndex]: {
68	isResolved: true,
69	resolutionsChecked: subBreach?.DataClasses \|\| [BreachDataTypes.General]	×
70	}
71	}
72
73	isV2Changed = true	×
74	}
75	if (v2[email][recencyIndex]?.isResolved) {	×
76	console.log(`recencyIndex ${recencyIndex} exists in v2 and is resolved, no changes`)	×
77	} else {
78	console.log(`recencyIndex ${recencyIndex} either does not exist or is not resolved, overwriting`)	×
79	v2[email][recencyIndex] = {	×
80	isResolved: true,
81	resolutionsChecked: subBreach?.DataClasses
82	}
83	isV2Changed = true	×
84	}
85	}
86	}
87
88	// check if v2 is changed, if so, upsert the new v2
89	if (isV2Changed) {	×
90	await setBreachResolution(subscriber, v2)	×
91	}
92	}
93	offset += LIMIT	×
94	} while (subscribersArr.length === LIMIT)
95
96	// breaking out of do..while loop
97	console.log('Reaching the end of the table, offset ended at', offset)	×
98	process.exit()	×

mozilla / blurts-server / b86637bb-ffe8-4e59-ade8-c8caf0aa8efc

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous