#12893

pending completion

Build # #12893

Build Type

push

circleci

Committed by

web-flow

Commit Message

Merge pull request #2923 from mozilla/MNTOR-1395

MNTOR-1395: kube job convert breach resolution script

Run Details

282 of 1527 branches covered (18.47%)

Branch coverage included in aggregate %.

73 of 73 new or added lines in 2 files covered. (100.0%)

959 of 4139 relevant lines covered (23.17%)

1.93 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/src/scripts/prod-experiments/benchmarkWithUpsert.js

/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

/**
 * Executes once
 * The purpose of the script is to convert all `subscriber.breaches_resolved` to `subscriber.breaches_resolution`
 * with the goal of deprecating the column
 */

import Knex from 'knex'
import knexConfig from '../../db/knexfile.js'
import { getAllBreachesFromDb } from '../../utils/hibp.js'
import { getAllEmailsAndBreaches } from '../../utils/breaches.js'
import { BreachDataTypes } from '../../utils/breach-resolution.js'
const knex = Knex(knexConfig)

const LIMIT = 1000 // with millions of records, we have to load a few at a time
let CAP = 5000 // cap the experiment
if (process.argv.length > 2) {
  CAP = process.argv[2]
  console.log('using cap passed in: ', CAP)
}
let offset = 0 // looping through all records with offset
let subscribersArr = []

/**
 * Batch update
 *
 * @param {*} updateCollection
 */
const batchUpdate = async (updateCollection) => {
  const trx = await knex.transaction()
  try {
    await Promise.all(updateCollection.map(tuple => {
      const { user, updatedBreachesResolution } = tuple
      return knex('subscribers')
        .where('id', user.id)
        .update({
          breach_resolution: updatedBreachesResolution
        })
        .transacting(trx)
    })
    )
    await trx.commit()
  } catch (error) {
    await trx.rollback()
    console.error('batch update failed!!')
    console.log({ updateCollection })
    console.error(error)
  }
}

const startTime = Date.now()
console.log(`Start time is: ${startTime}`)

// load all breaches for ref
const allBreaches = await getAllBreachesFromDb()
if (allBreaches && allBreaches.length > 0) console.log('breaches loaded successfully! ', allBreaches.length)

// find all subscribers who resolved any breaches in the past, convert those
// records into the new v2 format
do {
  console.log(`Converting breaches_resolved to breach_resolution - start: ${offset} limit: ${LIMIT}`)
  subscribersArr = await knex
    .select('id', 'primary_email', 'breaches_resolved', 'breach_resolution')
    .from('subscribers')
    .whereNotNull('breaches_resolved')
    .limit(LIMIT)
    .offset(offset)
    .orderBy('updated_at', 'desc')

  console.log(`Loaded # of subscribers: ${subscribersArr.length}`)
  const updateCollection = []

  for (const subscriber of subscribersArr) {
    let { breaches_resolved: v1, breach_resolution: v2 } = subscriber
    let isV2Changed = false // use a boolean to track if v2 has been changed, only upsert if so

    // fetch subscriber all breaches / email
    const subscriberBreachesEmail = await getAllEmailsAndBreaches(subscriber, allBreaches)
    // console.debug(JSON.stringify(subscriberBreachesEmail.verifiedEmails))

    for (const [email, resolvedRecencyIndices] of Object.entries(v1)) {
      // console.debug({ email })
      // console.debug({ resolvedRecencyIndices })
      for (const recencyIndex of resolvedRecencyIndices) {
        // console.debug({ recencyIndex })
        // find subscriber's relevant recency index breach information
        const ve = subscriberBreachesEmail.verifiedEmails?.filter(e => e.email === email)[0] || {}
        // console.debug({ ve })
        const subBreach = ve.breaches?.filter(b => Number(b.recencyIndex) === Number(recencyIndex))[0] || null
        // console.debug({ subBreach })

        if (!subBreach || !subBreach.DataClasses) {
          console.warn(`SKIP: Cannot find subscribers breach and data types - recency: ${recencyIndex} email: ${email}`)
          continue
        }

        // if email does not exist in v2, we need to add it to the object
        // format: {email: { recencyIndex: { isResolved: true, resolutionsChecked: [DataTypes]}}}
        if (!v2) v2 = {}
        if (!v2[email]) {
          v2[email] = {
            [recencyIndex]: {
              isResolved: true,
              resolutionsChecked: subBreach?.DataClasses || [BreachDataTypes.General]
            }
          }

          isV2Changed = true
        }
        if (v2[email][recencyIndex]?.isResolved) {
          console.log(`recencyIndex ${recencyIndex} exists in v2 and is resolved, no changes`)
        } else {
          console.log(`recencyIndex ${recencyIndex} either does not exist or is not resolved, overwriting`)
          v2[email][recencyIndex] = {
            isResolved: true,
            resolutionsChecked: subBreach?.DataClasses
          }
          isV2Changed = true
        }
      }
    }

    // check if v2 is changed, if so, upsert the new v2
    if (isV2Changed) {
      console.log('upsert for subscriber: ', subscriber.primary_email)
      updateCollection.push({ user: subscriber, updatedBreachesResolution: v2 })
    }
  }
  await batchUpdate(updateCollection)
  offset += LIMIT
} while (subscribersArr.length === LIMIT && offset <= CAP)

// breaking out of do..while loop
console.log('Reaching the end of the table, offset ended at', offset)
const endTime = Date.now()
console.log(`End time is: ${endTime}`)
console.log('Diff is: ', endTime - startTime)
process.exit()

1	/* This Source Code Form is subject to the terms of the Mozilla Public
2	* License, v. 2.0. If a copy of the MPL was not distributed with this
3	* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
4
5	/**
6	* Executes once
7	* The purpose of the script is to convert all `subscriber.breaches_resolved` to `subscriber.breaches_resolution`
8	* with the goal of deprecating the column
9	*/
10
11	import Knex from 'knex'
12	import knexConfig from '../../db/knexfile.js'
13	import { getAllBreachesFromDb } from '../../utils/hibp.js'
14	import { getAllEmailsAndBreaches } from '../../utils/breaches.js'
15	import { BreachDataTypes } from '../../utils/breach-resolution.js'
16	const knex = Knex(knexConfig)	×
17
18	const LIMIT = 1000 // with millions of records, we have to load a few at a time	×
19	let CAP = 5000 // cap the experiment	×
20	if (process.argv.length > 2) {	×
21	CAP = process.argv[2]	×
22	console.log('using cap passed in: ', CAP)	×
23	}
24	let offset = 0 // looping through all records with offset	×
25	let subscribersArr = []	×
26
27	/**
28	* Batch update
29	*
30	* @param {*} updateCollection
31	*/
32	const batchUpdate = async (updateCollection) => {	×
33	const trx = await knex.transaction()	×
34	try {	×
35	await Promise.all(updateCollection.map(tuple => {	×
36	const { user, updatedBreachesResolution } = tuple	×
37	return knex('subscribers')	×
38	.where('id', user.id)
39	.update({
40	breach_resolution: updatedBreachesResolution
41	})
42	.transacting(trx)
43	})
44	)
45	await trx.commit()	×
46	} catch (error) {
47	await trx.rollback()	×
48	console.error('batch update failed!!')	×
49	console.log({ updateCollection })	×
50	console.error(error)	×
51	}
52	}
53
54	const startTime = Date.now()	×
55	console.log(`Start time is: ${startTime}`)	×
56
57	// load all breaches for ref
58	const allBreaches = await getAllBreachesFromDb()	×
59	if (allBreaches && allBreaches.length > 0) console.log('breaches loaded successfully! ', allBreaches.length)	×
60
61	// find all subscribers who resolved any breaches in the past, convert those
62	// records into the new v2 format
63	do {	×
64	console.log(`Converting breaches_resolved to breach_resolution - start: ${offset} limit: ${LIMIT}`)	×
65	subscribersArr = await knex	×
66	.select('id', 'primary_email', 'breaches_resolved', 'breach_resolution')
67	.from('subscribers')
68	.whereNotNull('breaches_resolved')
69	.limit(LIMIT)
70	.offset(offset)
71	.orderBy('updated_at', 'desc')
72
73	console.log(`Loaded # of subscribers: ${subscribersArr.length}`)	×
74	const updateCollection = []	×
75
76	for (const subscriber of subscribersArr) {	×
77	let { breaches_resolved: v1, breach_resolution: v2 } = subscriber	×
78	let isV2Changed = false // use a boolean to track if v2 has been changed, only upsert if so	×
79
80	// fetch subscriber all breaches / email
81	const subscriberBreachesEmail = await getAllEmailsAndBreaches(subscriber, allBreaches)	×
82	// console.debug(JSON.stringify(subscriberBreachesEmail.verifiedEmails))
83
84	for (const [email, resolvedRecencyIndices] of Object.entries(v1)) {	×
85	// console.debug({ email })
86	// console.debug({ resolvedRecencyIndices })
87	for (const recencyIndex of resolvedRecencyIndices) {	×
88	// console.debug({ recencyIndex })
89	// find subscriber's relevant recency index breach information
90	const ve = subscriberBreachesEmail.verifiedEmails?.filter(e => e.email === email)[0] \|\| {}	×
91	// console.debug({ ve })
92	const subBreach = ve.breaches?.filter(b => Number(b.recencyIndex) === Number(recencyIndex))[0] \|\| null	×
93	// console.debug({ subBreach })
94
95	if (!subBreach \|\| !subBreach.DataClasses) {	×
96	console.warn(`SKIP: Cannot find subscribers breach and data types - recency: ${recencyIndex} email: ${email}`)	×
97	continue	×
98	}
99
100	// if email does not exist in v2, we need to add it to the object
101	// format: {email: { recencyIndex: { isResolved: true, resolutionsChecked: [DataTypes]}}}
102	if (!v2) v2 = {}	×
103	if (!v2[email]) {	×
104	v2[email] = {	×
105	[recencyIndex]: {
106	isResolved: true,
107	resolutionsChecked: subBreach?.DataClasses \|\| [BreachDataTypes.General]	×
108	}
109	}
110
111	isV2Changed = true	×
112	}
113	if (v2[email][recencyIndex]?.isResolved) {	×
114	console.log(`recencyIndex ${recencyIndex} exists in v2 and is resolved, no changes`)	×
115	} else {
116	console.log(`recencyIndex ${recencyIndex} either does not exist or is not resolved, overwriting`)	×
117	v2[email][recencyIndex] = {	×
118	isResolved: true,
119	resolutionsChecked: subBreach?.DataClasses
120	}
121	isV2Changed = true	×
122	}
123	}
124	}
125
126	// check if v2 is changed, if so, upsert the new v2
127	if (isV2Changed) {	×
128	console.log('upsert for subscriber: ', subscriber.primary_email)	×
129	updateCollection.push({ user: subscriber, updatedBreachesResolution: v2 })	×
130	}
131	}
132	await batchUpdate(updateCollection)	×
133	offset += LIMIT	×
134	} while (subscribersArr.length === LIMIT && offset <= CAP)	×
135
136	// breaking out of do..while loop
137	console.log('Reaching the end of the table, offset ended at', offset)	×
138	const endTime = Date.now()	×
139	console.log(`End time is: ${endTime}`)	×
140	console.log('Diff is: ', endTime - startTime)	×
141	process.exit()	×

mozilla / blurts-server / #12893

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous