#13004

pending completion

Build # #13004

Build Type

push

circleci

Committed by

web-flow

Commit Message

Merge pull request #2947 from mozilla/MNTOR-1395

MNTOR-1395: logs err if hibp fetch ever errs out

Run Details

282 of 1601 branches covered (17.61%)

Branch coverage included in aggregate %.

8 of 8 new or added lines in 2 files covered. (100.0%)

959 of 4322 relevant lines covered (22.19%)

1.85 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/src/scripts/kube-jobs/convertBreachResolutions.js

/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

/**
 * Executes once
 * The purpose of the script is to convert all `subscriber.breaches_resolved` to `subscriber.breaches_resolution`
 * with the goal of deprecating the column
 */

import Knex from 'knex'
import knexConfig from '../../db/knexfile.js'
import { getAllBreachesFromDb } from '../../utils/hibp.js'
import { getAllEmailsAndBreaches } from '../../utils/breaches.js'
import { BreachDataTypes } from '../../utils/breach-resolution.js'
const knex = Knex(knexConfig)

const LIMIT = 1000 // with millions of records, we have to load a few at a time
let subscribersArr = []

/**
 * Batch update
 *
 * @param {*} updateCollection
 */
const batchUpdate = async (updateCollection) => {
  const trx = await knex.transaction()
  try {
    await Promise.all(updateCollection.map(tuple => {
      const { user, updatedBreachesResolution } = tuple
      return knex('subscribers')
        .where('id', user.id)
        .update({
          breach_resolution: updatedBreachesResolution
        })
        .transacting(trx)
    }))
    await trx.commit()
  } catch (error) {
    await trx.rollback()
    console.error('batch update failed!!')
    console.log({ updateCollection })
    console.error(error)
  }
}

const selectAndLockResolutions = async () => {
  const trx = await knex.transaction()
  let subscribersArr = []
  try {
    subscribersArr = await knex.select('id', 'primary_email', 'breaches_resolved', 'breach_resolution')
      .from('subscribers')
      .whereNotNull('breaches_resolved')
      .whereNull('db_migration_1')
      .limit(LIMIT)
      .orderBy('updated_at', 'desc')
      .transacting(trx)
      .forUpdate()

    // update the lock
    await Promise.all(subscribersArr.map(sub => {
      const { id } = sub
      return knex('subscribers')
        .where('id', id)
        .update({
          db_migration_1: true
        })
        .transacting(trx)
    }))

    await trx.commit()
  } catch (error) {
    await trx.rollback()
    console.log('select & mark rows failed!! first row:')
    console.log({ first: subscribersArr[0] })
    console.error(error)
  }

  return subscribersArr
}

const startTime = Date.now()
console.log(`Start time is: ${startTime}`)

// load all breaches for ref
const allBreaches = await getAllBreachesFromDb()
if (allBreaches && allBreaches.length > 0) console.log('breaches loaded successfully! ', allBreaches.length)

// find all subscribers who resolved any breaches in the past, convert those
// records into the new v2 format
let failedToSelect = true
while (failedToSelect) {
  try {
    subscribersArr = await selectAndLockResolutions()
    failedToSelect = false
  } catch (e) {
    failedToSelect = true
    console.error(e)
  }
}

console.log(`Loaded # of subscribers: ${subscribersArr.length}`)
const updateCollection = []

for (const subscriber of subscribersArr) {
  let { breaches_resolved: v1, breach_resolution: v2 } = subscriber
  let isV2Changed = false // use a boolean to track if v2 has been changed, only upsert if so

  // fetch subscriber all breaches / email
  let subscriberBreachesEmail
  try {
    subscriberBreachesEmail = await getAllEmailsAndBreaches(subscriber, allBreaches)
  } catch (e) {
    console.error('Cannot fetch subscriber breaches at the moment: ', e)
    continue
  }
  // console.debug(JSON.stringify(subscriberBreachesEmail.verifiedEmails))

  for (const [email, resolvedRecencyIndices] of Object.entries(v1)) {
    // console.debug({ email })
    // console.debug({ resolvedRecencyIndices })
    for (const recencyIndex of resolvedRecencyIndices) {
      // console.debug({ recencyIndex })
      // find subscriber's relevant recency index breach information
      const ve = subscriberBreachesEmail.verifiedEmails?.filter(e => e.email === email)[0] || {}
      // console.debug({ ve })
      const subBreach = ve.breaches?.filter(b => Number(b.recencyIndex) === Number(recencyIndex))[0] || null
      // console.debug({ subBreach })

      if (!subBreach || !subBreach.DataClasses) {
        console.warn(`SKIP: Cannot find subscribers breach and data types - recency: ${recencyIndex} email: ${email}`)
        continue
      }

      // if email does not exist in v2, we need to add it to the object
      // format: {email: { recencyIndex: { isResolved: true, resolutionsChecked: [DataTypes]}}}
      if (!v2) v2 = {}
      if (!v2[email]) {
        v2[email] = {
          [recencyIndex]: {
            isResolved: true,
            resolutionsChecked: subBreach?.DataClasses || [BreachDataTypes.General]
          }
        }

        isV2Changed = true
      }
      if (v2[email][recencyIndex]?.isResolved) {
        console.log(`recencyIndex ${recencyIndex} exists in v2 and is resolved, no changes`)
      } else {
        console.log(`recencyIndex ${recencyIndex} either does not exist or is not resolved, overwriting`)
        v2[email][recencyIndex] = {
          isResolved: true,
          resolutionsChecked: subBreach?.DataClasses
        }
        isV2Changed = true
      }
    }
  }

  // check if v2 is changed, if so, upsert the new v2
  if (isV2Changed) {
    console.log('upsert for subscriber: ', subscriber.primary_email)
    updateCollection.push({ user: subscriber, updatedBreachesResolution: v2 })
  }
}
await batchUpdate(updateCollection)

// breaking out of do..while loop
console.log('Script finished')
const endTime = Date.now()
console.log(`End time is: ${endTime}`)
console.log('Diff is: ', endTime - startTime)
process.exit()

1	/* This Source Code Form is subject to the terms of the Mozilla Public
2	* License, v. 2.0. If a copy of the MPL was not distributed with this
3	* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
4
5	/**
6	* Executes once
7	* The purpose of the script is to convert all `subscriber.breaches_resolved` to `subscriber.breaches_resolution`
8	* with the goal of deprecating the column
9	*/
10
11	import Knex from 'knex'
12	import knexConfig from '../../db/knexfile.js'
13	import { getAllBreachesFromDb } from '../../utils/hibp.js'
14	import { getAllEmailsAndBreaches } from '../../utils/breaches.js'
15	import { BreachDataTypes } from '../../utils/breach-resolution.js'
16	const knex = Knex(knexConfig)	×
17
18	const LIMIT = 1000 // with millions of records, we have to load a few at a time	×
19	let subscribersArr = []	×
20
21	/**
22	* Batch update
23	*
24	* @param {*} updateCollection
25	*/
26	const batchUpdate = async (updateCollection) => {	×
27	const trx = await knex.transaction()	×
28	try {	×
29	await Promise.all(updateCollection.map(tuple => {	×
30	const { user, updatedBreachesResolution } = tuple	×
31	return knex('subscribers')	×
32	.where('id', user.id)
33	.update({
34	breach_resolution: updatedBreachesResolution
35	})
36	.transacting(trx)
37	}))
38	await trx.commit()	×
39	} catch (error) {
40	await trx.rollback()	×
41	console.error('batch update failed!!')	×
42	console.log({ updateCollection })	×
43	console.error(error)	×
44	}
45	}
46
47	const selectAndLockResolutions = async () => {	×
48	const trx = await knex.transaction()	×
49	let subscribersArr = []	×
50	try {	×
51	subscribersArr = await knex.select('id', 'primary_email', 'breaches_resolved', 'breach_resolution')	×
52	.from('subscribers')
53	.whereNotNull('breaches_resolved')
54	.whereNull('db_migration_1')
55	.limit(LIMIT)
56	.orderBy('updated_at', 'desc')
57	.transacting(trx)
58	.forUpdate()
59
60	// update the lock
61	await Promise.all(subscribersArr.map(sub => {	×
62	const { id } = sub	×
63	return knex('subscribers')	×
64	.where('id', id)
65	.update({
66	db_migration_1: true
67	})
68	.transacting(trx)
69	}))
70
71	await trx.commit()	×
72	} catch (error) {
73	await trx.rollback()	×
74	console.log('select & mark rows failed!! first row:')	×
75	console.log({ first: subscribersArr[0] })	×
76	console.error(error)	×
77	}
78
79	return subscribersArr	×
80	}
81
82	const startTime = Date.now()	×
83	console.log(`Start time is: ${startTime}`)	×
84
85	// load all breaches for ref
86	const allBreaches = await getAllBreachesFromDb()	×
87	if (allBreaches && allBreaches.length > 0) console.log('breaches loaded successfully! ', allBreaches.length)	×
88
89	// find all subscribers who resolved any breaches in the past, convert those
90	// records into the new v2 format
91	let failedToSelect = true	×
92	while (failedToSelect) {	×
93	try {	×
94	subscribersArr = await selectAndLockResolutions()	×
95	failedToSelect = false	×
96	} catch (e) {
97	failedToSelect = true	×
98	console.error(e)	×
99	}
100	}
101
102	console.log(`Loaded # of subscribers: ${subscribersArr.length}`)	×
103	const updateCollection = []	×
104
105	for (const subscriber of subscribersArr) {	×
106	let { breaches_resolved: v1, breach_resolution: v2 } = subscriber	×
107	let isV2Changed = false // use a boolean to track if v2 has been changed, only upsert if so	×
108
109	// fetch subscriber all breaches / email
110	let subscriberBreachesEmail
111	try {	×
112	subscriberBreachesEmail = await getAllEmailsAndBreaches(subscriber, allBreaches)	×
113	} catch (e) {
114	console.error('Cannot fetch subscriber breaches at the moment: ', e)	×
115	continue	×
116	}
117	// console.debug(JSON.stringify(subscriberBreachesEmail.verifiedEmails))
118
119	for (const [email, resolvedRecencyIndices] of Object.entries(v1)) {	×
120	// console.debug({ email })
121	// console.debug({ resolvedRecencyIndices })
122	for (const recencyIndex of resolvedRecencyIndices) {	×
123	// console.debug({ recencyIndex })
124	// find subscriber's relevant recency index breach information
125	const ve = subscriberBreachesEmail.verifiedEmails?.filter(e => e.email === email)[0] \|\| {}	×
126	// console.debug({ ve })
127	const subBreach = ve.breaches?.filter(b => Number(b.recencyIndex) === Number(recencyIndex))[0] \|\| null	×
128	// console.debug({ subBreach })
129
130	if (!subBreach \|\| !subBreach.DataClasses) {	×
131	console.warn(`SKIP: Cannot find subscribers breach and data types - recency: ${recencyIndex} email: ${email}`)	×
132	continue	×
133	}
134
135	// if email does not exist in v2, we need to add it to the object
136	// format: {email: { recencyIndex: { isResolved: true, resolutionsChecked: [DataTypes]}}}
137	if (!v2) v2 = {}	×
138	if (!v2[email]) {	×
139	v2[email] = {	×
140	[recencyIndex]: {
141	isResolved: true,
142	resolutionsChecked: subBreach?.DataClasses \|\| [BreachDataTypes.General]	×
143	}
144	}
145
146	isV2Changed = true	×
147	}
148	if (v2[email][recencyIndex]?.isResolved) {	×
149	console.log(`recencyIndex ${recencyIndex} exists in v2 and is resolved, no changes`)	×
150	} else {
151	console.log(`recencyIndex ${recencyIndex} either does not exist or is not resolved, overwriting`)	×
152	v2[email][recencyIndex] = {	×
153	isResolved: true,
154	resolutionsChecked: subBreach?.DataClasses
155	}
156	isV2Changed = true	×
157	}
158	}
159	}
160
161	// check if v2 is changed, if so, upsert the new v2
162	if (isV2Changed) {	×
163	console.log('upsert for subscriber: ', subscriber.primary_email)	×
164	updateCollection.push({ user: subscriber, updatedBreachesResolution: v2 })	×
165	}
166	}
167	await batchUpdate(updateCollection)	×
168
169	// breaking out of do..while loop
170	console.log('Script finished')	×
171	const endTime = Date.now()	×
172	console.log(`End time is: ${endTime}`)	×
173	console.log('Diff is: ', endTime - startTime)	×
174	process.exit()	×

mozilla / blurts-server / #13004

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous