83a5c888-89d1-42a2-b433-e63207c23b9a

Build # 83a5c888-89d1-42a2-b433-e63207c23b9a

Build Type

push

circleci

Committed by Joey Zhou

Commit Message

modify migration script

Run Details

282 of 1631 branches covered (17.29%)

Branch coverage included in aggregate %.

28 of 28 new or added lines in 1 file covered. (100.0%)

959 of 4373 relevant lines covered (21.93%)

3.66 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/src/scripts/migrationCleanup.js

/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

/**
 * Executes once
 * The purpose of the script is to clean up some of the failed records during db migration on 3/28/23
 */

import Knex from 'knex'
import knexConfig from '../db/knexfile.js'
import { getAllBreachesFromDb } from '../utils/hibp.js'
import { getAllEmailsAndBreaches } from '../utils/breaches.js'
import { setBreachResolution } from '../db/tables/subscribers.js'
import mozlog from '../utils/log.js'
const log = mozlog('script.migrationCleanup')
const knex = Knex(knexConfig)

let subscribersArr = []
// load all breaches for ref
const allBreaches = await getAllBreachesFromDb()
if (allBreaches && allBreaches.length > 0) log.info('breach_count', 'breaches loaded successfully! ', allBreaches.length)

const count = await knex
  .from('subscribers')
  .whereRaw('NOT ((breach_resolution)::jsonb \\? \'useBreachId\')')
  .count('*')

log.info('total_to_be_executed', count[0])

// find all subscribers who resolved any breaches in the past,
// replace recency index with breach id

subscribersArr = await knex
  .select('id', 'primary_email', 'breach_resolution')
  .from('subscribers')
  .whereRaw('NOT ((breach_resolution)::jsonb \\? \'useBreachId\')')

log.info('job', `Loaded # of subscribers: ${subscribersArr.length}`)

for (const subscriber of subscribersArr) {
  const { breach_resolution: v2 } = subscriber
  // console.debug({ v2 })

  // if useBreachId is set, skip because this breach_resolution has already been worked on
  if (v2.useBreachId) {
    log.warn('job', 'Skipping since `useBreachId` is set already, this breach resolution is already converted')
    continue
  }

  const newResolutions = {}

  // fetch subscriber all breaches / email
  const subscriberBreachesEmail = await getAllEmailsAndBreaches(subscriber, allBreaches)
  // console.debug(JSON.stringify(subscriberBreachesEmail.verifiedEmails))

  for (const email in v2) {
    // console.debug({ email })
    const resolutions = v2[email]
    // console.debug({ resolutions })
    newResolutions[email] = {}

    for (const recencyIndex in resolutions) {
      console.debug({ recencyIndex })

      // find subscriber's relevant recency index breach information
      const ve = subscriberBreachesEmail.verifiedEmails?.filter(ve => ve.email === email)[0] || {}
      const subBreach = ve.breaches?.filter(b => Number(b.recencyIndex) === Number(recencyIndex))[0] || null
      const breachName = subBreach?.Name
      console.debug({ breachName })

      // find breach id for the breach
      const breachId = allBreaches.find(b => b.Name === breachName)?.Id
      log.info('job', { breachId })
      newResolutions[email][breachId] = v2[email][recencyIndex]
    }
  }

  // check if v2 is changed, if so, upsert the new v2
  newResolutions.useBreachId = true
  await setBreachResolution(subscriber, newResolutions)
}

// breaking out of do..while loop
log.info('job', 'Reaching the end of the table')
process.exit()

1	/* This Source Code Form is subject to the terms of the Mozilla Public
2	* License, v. 2.0. If a copy of the MPL was not distributed with this
3	* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
4
5	/**
6	* Executes once
7	* The purpose of the script is to clean up some of the failed records during db migration on 3/28/23
8	*/
9
10	import Knex from 'knex'
11	import knexConfig from '../db/knexfile.js'
12	import { getAllBreachesFromDb } from '../utils/hibp.js'
13	import { getAllEmailsAndBreaches } from '../utils/breaches.js'
14	import { setBreachResolution } from '../db/tables/subscribers.js'
15	import mozlog from '../utils/log.js'
16	const log = mozlog('script.migrationCleanup')	×
17	const knex = Knex(knexConfig)	×
18
19	let subscribersArr = []	×
20	// load all breaches for ref
21	const allBreaches = await getAllBreachesFromDb()	×
22	if (allBreaches && allBreaches.length > 0) log.info('breach_count', 'breaches loaded successfully! ', allBreaches.length)	×
23
24	const count = await knex	×
25	.from('subscribers')
26	.whereRaw('NOT ((breach_resolution)::jsonb \\? \'useBreachId\')')
27	.count('*')
28
29	log.info('total_to_be_executed', count[0])	×
30
31	// find all subscribers who resolved any breaches in the past,
32	// replace recency index with breach id
33
34	subscribersArr = await knex	×
35	.select('id', 'primary_email', 'breach_resolution')
36	.from('subscribers')
37	.whereRaw('NOT ((breach_resolution)::jsonb \\? \'useBreachId\')')
38
39	log.info('job', `Loaded # of subscribers: ${subscribersArr.length}`)	×
40
41	for (const subscriber of subscribersArr) {	×
42	const { breach_resolution: v2 } = subscriber	×
43	// console.debug({ v2 })
44
45	// if useBreachId is set, skip because this breach_resolution has already been worked on
46	if (v2.useBreachId) {	×
47	log.warn('job', 'Skipping since `useBreachId` is set already, this breach resolution is already converted')	×
48	continue	×
49	}
50
51	const newResolutions = {}	×
52
53	// fetch subscriber all breaches / email
54	const subscriberBreachesEmail = await getAllEmailsAndBreaches(subscriber, allBreaches)	×
55	// console.debug(JSON.stringify(subscriberBreachesEmail.verifiedEmails))
56
57	for (const email in v2) {	×
58	// console.debug({ email })
59	const resolutions = v2[email]	×
60	// console.debug({ resolutions })
61	newResolutions[email] = {}	×
62
63	for (const recencyIndex in resolutions) {	×
64	console.debug({ recencyIndex })	×
65
66	// find subscriber's relevant recency index breach information
67	const ve = subscriberBreachesEmail.verifiedEmails?.filter(ve => ve.email === email)[0] \|\| {}	×
68	const subBreach = ve.breaches?.filter(b => Number(b.recencyIndex) === Number(recencyIndex))[0] \|\| null	×
69	const breachName = subBreach?.Name	×
70	console.debug({ breachName })	×
71
72	// find breach id for the breach
73	const breachId = allBreaches.find(b => b.Name === breachName)?.Id	×
74	log.info('job', { breachId })	×
75	newResolutions[email][breachId] = v2[email][recencyIndex]	×
76	}
77	}
78
79	// check if v2 is changed, if so, upsert the new v2
80	newResolutions.useBreachId = true	×
81	await setBreachResolution(subscriber, newResolutions)	×
82	}
83
84	// breaking out of do..while loop
85	log.info('job', 'Reaching the end of the table')	×
86	process.exit()	×

mozilla / blurts-server / 83a5c888-89d1-42a2-b433-e63207c23b9a

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous