#13107

pending completion

Build # #13107

Build Type

push

circleci

Committed by

mansaj

Commit Message

migration cleanup script

Run Details

282 of 1613 branches covered (17.48%)

Branch coverage included in aggregate %.

36 of 36 new or added lines in 1 file covered. (100.0%)

959 of 4367 relevant lines covered (21.96%)

1.83 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/src/scripts/migrationCleanup.js

/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

/**
 * Executes once
 * The purpose of the script is to clean up some of the failed records during db migration on 3/28/23
 */

import Knex from 'knex'
import knexConfig from '../db/knexfile.js'
import { getAllBreachesFromDb } from '../utils/hibp.js'
import { getAllEmailsAndBreaches } from '../utils/breaches.js'
import { setBreachResolution } from '../db/tables/subscribers.js'
const knex = Knex(knexConfig)

const LIMIT = 1000 // with millions of records, we have to load a few at a time
let offset = 0 // looping through all records with offset
let subscribersArr = []

// load all breaches for ref
const allBreaches = await getAllBreachesFromDb()
if (allBreaches && allBreaches.length > 0) console.log('breaches loaded successfully! ', allBreaches.length)
console.log(JSON.stringify(allBreaches[0]))

// find all subscribers who resolved any breaches in the past,
// replace recency index with breach id
do {
  console.log(`Converting breach_resolution to use breach Id - start: ${offset} limit: ${LIMIT}`)
  subscribersArr = await knex
    .select('id', 'primary_email', 'breach_resolution')
    .from('subscribers')
    .whereRaw('NOT ((breach_resolution)::jsonb \\? \'useBreachId\')')
    .limit(LIMIT)
    .offset(offset)

  console.log(`Loaded # of subscribers: ${subscribersArr.length}`)

  for (const subscriber of subscribersArr) {
    const { breach_resolution: v2 } = subscriber
    console.debug({ v2 })

    // if useBreachId is set, skip because this breach_resolution has already been worked on
    if (v2.useBreachId) {
      console.log('Skipping since `useBreachId` is set already, this breach resolution is already converted')
      continue
    }

    const newResolutions = {}

    // fetch subscriber all breaches / email
    const subscriberBreachesEmail = await getAllEmailsAndBreaches(subscriber, allBreaches)
    // console.debug(JSON.stringify(subscriberBreachesEmail.verifiedEmails))

    for (const email in v2) {
      // console.debug({ email })
      const resolutions = v2[email]
      // console.debug({ resolutions })
      newResolutions[email] = {}

      for (const recencyIndex in resolutions) {
        console.debug({ recencyIndex })

        // find subscriber's relevant recency index breach information
        const ve = subscriberBreachesEmail.verifiedEmails?.filter(ve => ve.email === email)[0] || {}
        const subBreach = ve.breaches?.filter(b => Number(b.recencyIndex) === Number(recencyIndex))[0] || null
        const breachName = subBreach?.Name
        console.debug({ breachName })

        // find breach id for the breach
        const breachId = allBreaches.find(b => b.Name === breachName)?.Id
        console.log({ breachId })
        newResolutions[email][breachId] = v2[email][recencyIndex]
      }
    }

    // check if v2 is changed, if so, upsert the new v2
    newResolutions.useBreachId = true
    // console.log(JSON.stringify(newResolutions))
    await setBreachResolution(subscriber, newResolutions)
  }
  offset += LIMIT
} while (subscribersArr.length === LIMIT)

// breaking out of do..while loop
console.log('Reaching the end of the table, offset ended at', offset)
process.exit()

1	/* This Source Code Form is subject to the terms of the Mozilla Public
2	* License, v. 2.0. If a copy of the MPL was not distributed with this
3	* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
4
5	/**
6	* Executes once
7	* The purpose of the script is to clean up some of the failed records during db migration on 3/28/23
8	*/
9
10	import Knex from 'knex'
11	import knexConfig from '../db/knexfile.js'
12	import { getAllBreachesFromDb } from '../utils/hibp.js'
13	import { getAllEmailsAndBreaches } from '../utils/breaches.js'
14	import { setBreachResolution } from '../db/tables/subscribers.js'
15	const knex = Knex(knexConfig)	×
16
17	const LIMIT = 1000 // with millions of records, we have to load a few at a time	×
18	let offset = 0 // looping through all records with offset	×
19	let subscribersArr = []	×
20
21	// load all breaches for ref
22	const allBreaches = await getAllBreachesFromDb()	×
23	if (allBreaches && allBreaches.length > 0) console.log('breaches loaded successfully! ', allBreaches.length)	×
24	console.log(JSON.stringify(allBreaches[0]))	×
25
26	// find all subscribers who resolved any breaches in the past,
27	// replace recency index with breach id
28	do {	×
29	console.log(`Converting breach_resolution to use breach Id - start: ${offset} limit: ${LIMIT}`)	×
30	subscribersArr = await knex	×
31	.select('id', 'primary_email', 'breach_resolution')
32	.from('subscribers')
33	.whereRaw('NOT ((breach_resolution)::jsonb \\? \'useBreachId\')')
34	.limit(LIMIT)
35	.offset(offset)
36
37	console.log(`Loaded # of subscribers: ${subscribersArr.length}`)	×
38
39	for (const subscriber of subscribersArr) {	×
40	const { breach_resolution: v2 } = subscriber	×
41	console.debug({ v2 })	×
42
43	// if useBreachId is set, skip because this breach_resolution has already been worked on
44	if (v2.useBreachId) {	×
45	console.log('Skipping since `useBreachId` is set already, this breach resolution is already converted')	×
46	continue	×
47	}
48
49	const newResolutions = {}	×
50
51	// fetch subscriber all breaches / email
52	const subscriberBreachesEmail = await getAllEmailsAndBreaches(subscriber, allBreaches)	×
53	// console.debug(JSON.stringify(subscriberBreachesEmail.verifiedEmails))
54
55	for (const email in v2) {	×
56	// console.debug({ email })
57	const resolutions = v2[email]	×
58	// console.debug({ resolutions })
59	newResolutions[email] = {}	×
60
61	for (const recencyIndex in resolutions) {	×
62	console.debug({ recencyIndex })	×
63
64	// find subscriber's relevant recency index breach information
65	const ve = subscriberBreachesEmail.verifiedEmails?.filter(ve => ve.email === email)[0] \|\| {}	×
66	const subBreach = ve.breaches?.filter(b => Number(b.recencyIndex) === Number(recencyIndex))[0] \|\| null	×
67	const breachName = subBreach?.Name	×
68	console.debug({ breachName })	×
69
70	// find breach id for the breach
71	const breachId = allBreaches.find(b => b.Name === breachName)?.Id	×
72	console.log({ breachId })	×
73	newResolutions[email][breachId] = v2[email][recencyIndex]	×
74	}
75	}
76
77	// check if v2 is changed, if so, upsert the new v2
78	newResolutions.useBreachId = true	×
79	// console.log(JSON.stringify(newResolutions))
80	await setBreachResolution(subscriber, newResolutions)	×
81	}
82	offset += LIMIT	×
83	} while (subscribersArr.length === LIMIT)
84
85	// breaking out of do..while loop
86	console.log('Reaching the end of the table, offset ended at', offset)	×
87	process.exit()	×

mozilla / blurts-server / #13107

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous