#12818

pending completion

Build # #12818

Build Type

push

circleci

Committed by

web-flow

Commit Message

Merge pull request #2911 from mozilla/MNTOR-1395

MNTOR-1395: Some benchmark tests for prod migration

Run Details

282 of 1467 branches covered (19.22%)

Branch coverage included in aggregate %.

76 of 76 new or added lines in 3 files covered. (100.0%)

959 of 3992 relevant lines covered (24.02%)

2.0 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/src/scripts/prod-experiments/benchmarkNoUpsert.js

/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

/**
 * Executes once
 * The purpose of the script is to convert all `subscriber.breaches_resolved` to `subscriber.breaches_resolution`
 * with the goal of deprecating the column
 */

import Knex from 'knex'
import knexConfig from '../../db/knexfile.js'
import { getAllBreachesFromDb } from '../../utils/hibp.js'
import { getAllEmailsAndBreaches } from '../../utils/breaches.js'
import { BreachDataTypes } from '../../utils/breach-resolution.js'
const knex = Knex(knexConfig)

const LIMIT = 100 // with millions of records, we have to load a few at a time
let offset = 0 // looping through all records with offset
let subscribersArr = []

const startTime = Date.now()
console.log(`Start time is: ${startTime}`)

// load all breaches for ref
const allBreaches = await getAllBreachesFromDb()
if (allBreaches && allBreaches.length > 0) console.log('breaches loaded successfully! ', allBreaches.length)

// find all subscribers who resolved any breaches in the past, convert those
// records into the new v2 format
do {
  console.log(`Converting breaches_resolved to breach_resolution - start: ${offset} limit: ${LIMIT}`)
  subscribersArr = await knex
    .select('id', 'primary_email', 'breaches_resolved', 'breach_resolution')
    .from('subscribers')
    .whereNotNull('breaches_resolved')
    .limit(LIMIT)
    .offset(offset)

  console.log(`Loaded # of subscribers: ${subscribersArr.length}`)

  for (const subscriber of subscribersArr) {
    let { breaches_resolved: v1, breach_resolution: v2 } = subscriber

    let isV2Changed = false // use a boolean to track if v2 has been changed, only upsert if so

    // fetch subscriber all breaches / email
    const subscriberBreachesEmail = await getAllEmailsAndBreaches(subscriber, allBreaches)
    // console.debug(JSON.stringify(subscriberBreachesEmail.verifiedEmails))

    for (const [email, resolvedRecencyIndices] of Object.entries(v1)) {
      // console.debug({ email })
      // console.debug({ resolvedRecencyIndices })
      for (const recencyIndex of resolvedRecencyIndices) {
        // console.debug({ recencyIndex })
        // find subscriber's relevant recency index breach information
        const ve = subscriberBreachesEmail.verifiedEmails?.filter(ve => ve.email === email)[0] || {}
        const subBreach = ve.breaches?.filter(b => Number(b.recencyIndex) === Number(recencyIndex))[0] || null
        // console.debug({ subBreach })

        if (!subBreach || !subBreach.DataClasses) {
          console.warn(`SKIP: Cannot find subscribers breach and data types - recency: ${recencyIndex} email: ${email}`)
          continue
        }

        // if email does not exist in v2, we need to add it to the object
        // format: {email: { recencyIndex: { isResolved: true, resolutionsChecked: [DataTypes]}}}
        if (!v2) v2 = {}
        if (!v2[email]) {
          v2[email] = {
            [recencyIndex]: {
              isResolved: true,
              resolutionsChecked: subBreach?.DataClasses || [BreachDataTypes.General]
            }
          }

          isV2Changed = true
        }
        if (v2[email][recencyIndex]?.isResolved) {
          console.log(`recencyIndex ${recencyIndex} exists in v2 and is resolved, no changes`)
        } else {
          console.log(`recencyIndex ${recencyIndex} either does not exist or is not resolved, overwriting`)
          v2[email][recencyIndex] = {
            isResolved: true,
            resolutionsChecked: subBreach?.DataClasses
          }
          isV2Changed = true
        }
      }
    }

    // check if v2 is changed, if so, upsert the new v2
    if (isV2Changed) {
      console.log('upsert for subscriber: ', subscriber.primary_email)
    }
  }
  offset += LIMIT
} while (subscribersArr.length === LIMIT && offset <= 50000)

// breaking out of do..while loop
console.log('Reaching the end of the table, offset ended at', offset)
const endTime = Date.now()
console.log(`End time is: ${endTime}`)
console.log('Diff is: ', endTime - startTime)
process.exit()

1	/* This Source Code Form is subject to the terms of the Mozilla Public
2	* License, v. 2.0. If a copy of the MPL was not distributed with this
3	* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
4
5	/**
6	* Executes once
7	* The purpose of the script is to convert all `subscriber.breaches_resolved` to `subscriber.breaches_resolution`
8	* with the goal of deprecating the column
9	*/
10
11	import Knex from 'knex'
12	import knexConfig from '../../db/knexfile.js'
13	import { getAllBreachesFromDb } from '../../utils/hibp.js'
14	import { getAllEmailsAndBreaches } from '../../utils/breaches.js'
15	import { BreachDataTypes } from '../../utils/breach-resolution.js'
16	const knex = Knex(knexConfig)	×
17
18	const LIMIT = 100 // with millions of records, we have to load a few at a time	×
19	let offset = 0 // looping through all records with offset	×
20	let subscribersArr = []	×
21
22	const startTime = Date.now()	×
23	console.log(`Start time is: ${startTime}`)	×
24
25	// load all breaches for ref
26	const allBreaches = await getAllBreachesFromDb()	×
27	if (allBreaches && allBreaches.length > 0) console.log('breaches loaded successfully! ', allBreaches.length)	×
28
29	// find all subscribers who resolved any breaches in the past, convert those
30	// records into the new v2 format
31	do {	×
32	console.log(`Converting breaches_resolved to breach_resolution - start: ${offset} limit: ${LIMIT}`)	×
33	subscribersArr = await knex	×
34	.select('id', 'primary_email', 'breaches_resolved', 'breach_resolution')
35	.from('subscribers')
36	.whereNotNull('breaches_resolved')
37	.limit(LIMIT)
38	.offset(offset)
39
40	console.log(`Loaded # of subscribers: ${subscribersArr.length}`)	×
41
42	for (const subscriber of subscribersArr) {	×
43	let { breaches_resolved: v1, breach_resolution: v2 } = subscriber	×
44
45	let isV2Changed = false // use a boolean to track if v2 has been changed, only upsert if so	×
46
47	// fetch subscriber all breaches / email
48	const subscriberBreachesEmail = await getAllEmailsAndBreaches(subscriber, allBreaches)	×
49	// console.debug(JSON.stringify(subscriberBreachesEmail.verifiedEmails))
50
51	for (const [email, resolvedRecencyIndices] of Object.entries(v1)) {	×
52	// console.debug({ email })
53	// console.debug({ resolvedRecencyIndices })
54	for (const recencyIndex of resolvedRecencyIndices) {	×
55	// console.debug({ recencyIndex })
56	// find subscriber's relevant recency index breach information
57	const ve = subscriberBreachesEmail.verifiedEmails?.filter(ve => ve.email === email)[0] \|\| {}	×
58	const subBreach = ve.breaches?.filter(b => Number(b.recencyIndex) === Number(recencyIndex))[0] \|\| null	×
59	// console.debug({ subBreach })
60
61	if (!subBreach \|\| !subBreach.DataClasses) {	×
62	console.warn(`SKIP: Cannot find subscribers breach and data types - recency: ${recencyIndex} email: ${email}`)	×
63	continue	×
64	}
65
66	// if email does not exist in v2, we need to add it to the object
67	// format: {email: { recencyIndex: { isResolved: true, resolutionsChecked: [DataTypes]}}}
68	if (!v2) v2 = {}	×
69	if (!v2[email]) {	×
70	v2[email] = {	×
71	[recencyIndex]: {
72	isResolved: true,
73	resolutionsChecked: subBreach?.DataClasses \|\| [BreachDataTypes.General]	×
74	}
75	}
76
77	isV2Changed = true	×
78	}
79	if (v2[email][recencyIndex]?.isResolved) {	×
80	console.log(`recencyIndex ${recencyIndex} exists in v2 and is resolved, no changes`)	×
81	} else {
82	console.log(`recencyIndex ${recencyIndex} either does not exist or is not resolved, overwriting`)	×
83	v2[email][recencyIndex] = {	×
84	isResolved: true,
85	resolutionsChecked: subBreach?.DataClasses
86	}
87	isV2Changed = true	×
88	}
89	}
90	}
91
92	// check if v2 is changed, if so, upsert the new v2
93	if (isV2Changed) {	×
94	console.log('upsert for subscriber: ', subscriber.primary_email)	×
95	}
96	}
97	offset += LIMIT	×
98	} while (subscribersArr.length === LIMIT && offset <= 50000)	×
99
100	// breaking out of do..while loop
101	console.log('Reaching the end of the table, offset ended at', offset)	×
102	const endTime = Date.now()	×
103	console.log(`End time is: ${endTime}`)	×
104	console.log('Diff is: ', endTime - startTime)	×
105	process.exit()	×

mozilla / blurts-server / #12818

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous