#13037

pending completion

Build # #13037

Build Type

push

circleci

Committed by

web-flow

Commit Message

Merge pull request #2955 from mozilla/MNTOR-1395

MNTOR-1395: simplify where statement

Run Details

282 of 1601 branches covered (17.61%)

Branch coverage included in aggregate %.

959 of 4323 relevant lines covered (22.18%)

1.85 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/src/scripts/kube-jobs/recencyToBreachId.js

/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

/**
 * Executes once
 * Execute after `deprecateBreachResolve.js`
 * The purpose of the script is to convert all recency indices to breach ids for `breach_resolution`
 * For backwards compatibility, all converted `breach_resolution` fields will have a boolean
 * `useBreachId: true/false`
 */

import Knex from 'knex'
import knexConfig from '../../db/knexfile.js'
import { getAllBreachesFromDb } from '../../utils/hibp.js'
import { getAllEmailsAndBreaches } from '../../utils/breaches.js'
const knex = Knex(knexConfig)

const LIMIT = 1000 // with millions of records, we have to load a few at a time
let subscribersArr = []

const selectAndLockResolutions = async () => {
  const trx = await knex.transaction()
  let subscribers = []
  try {
    subscribers = await knex.select('id', 'primary_email', 'breach_resolution')
      .from('subscribers')
      .whereNotNull('breach_resolution')
      .whereNull('db_migration_2')
      .limit(LIMIT)
      .orderBy('updated_at', 'desc')
      .transacting(trx)
      .forUpdate()

    // update the lock
    await Promise.all(subscribers.map(sub => {
      const { id } = sub
      return knex('subscribers')
        .where('id', id)
        .update({
          db_migration_2: true
        })
        .transacting(trx)
    }))

    await trx.commit()
  } catch (error) {
    await trx.rollback()
    console.error('select & mark rows failed!! first row:')
    console.log({ first: subscribers[0] })
    console.error(error)
  }

  return subscribers
}

/**
 * Batch update
 *
 * @param {*} updateCollection
 */
const batchUpdate = async (updateCollection) => {
  const trx = await knex.transaction()
  try {
    await Promise.all(updateCollection.map(tuple => {
      const { user, updatedBreachesResolution } = tuple
      return knex('subscribers')
        .where('id', user.id)
        .update({
          breach_resolution: updatedBreachesResolution
        })
        .transacting(trx)
    }))
    await trx.commit()
  } catch (error) {
    await trx.rollback()
    console.error('batch update failed!!')
    console.log({ updateCollection })
    console.error(error)
  }
}

// Script begins here
const startTime = Date.now()
console.log(`Start time is: ${startTime}`)

// load all breaches for ref
const allBreaches = await getAllBreachesFromDb()
if (allBreaches && allBreaches.length > 0) console.log('breaches loaded successfully! ', allBreaches.length)
// console.log(JSON.stringify(allBreaches[0]))

// find all subscribers who resolved any breaches in the past,
// replace recency index with breach id

let failedToSelect = true
while (failedToSelect) {
  try {
    subscribersArr = await selectAndLockResolutions()
    failedToSelect = false
  } catch (e) {
    console.error(e)
  }
}

console.log(`Loaded # of subscribers: ${subscribersArr.length}`)
const updateCollection = []

for (const subscriber of subscribersArr) {
  const { breach_resolution: v2 } = subscriber
  // console.debug({ v2 })

  // if useBreachId is set, skip because this breach_resolution has already been worked on
  if (v2.useBreachId) {
    console.log('Skipping since `useBreachId` is set already, this breach resolution is already converted')
    continue
  }

  const newResolutions = {}

  // fetch subscriber all breaches / email
  let subscriberBreachesEmail
  try {
    subscriberBreachesEmail = await getAllEmailsAndBreaches(subscriber, allBreaches)
  } catch (e) {
    console.error('Cannot fetch subscriber breaches at the moment: ', e)
    continue
  }
  // console.debug(JSON.stringify(subscriberBreachesEmail.verifiedEmails))

  for (const email in v2) {
    // console.debug({ email })
    const resolutions = v2[email]
    // console.debug({ resolutions })
    newResolutions[email] = {}

    for (const recencyIndex in resolutions) {
      // console.debug({ recencyIndex })

      // find subscriber's relevant recency index breach information
      const ve = subscriberBreachesEmail.verifiedEmails?.filter(ve => ve.email === email)[0] || {}
      const subBreach = ve.breaches?.filter(b => Number(b.recencyIndex) === Number(recencyIndex))[0] || null
      const breachName = subBreach?.Name
      console.debug({ breachName })

      // find breach id for the breach
      const breachId = allBreaches.find(b => b.Name === breachName)?.Id
      console.log({ breachId })
      newResolutions[email][breachId] = v2[email][recencyIndex]
    }
  }

  // check if v2 is changed, if so, upsert the new v2
  newResolutions.useBreachId = true
  // console.log(JSON.stringify(newResolutions))
  updateCollection.push({ user: subscriber, updatedBreachesResolution: newResolutions })
}

await batchUpdate(updateCollection)

console.log('Reaching the end of the table')
const endTime = Date.now()
console.log(`End time is: ${endTime}`)
console.log('Diff is: ', endTime - startTime)
process.exit()

1	/* This Source Code Form is subject to the terms of the Mozilla Public
2	* License, v. 2.0. If a copy of the MPL was not distributed with this
3	* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
4
5	/**
6	* Executes once
7	* Execute after `deprecateBreachResolve.js`
8	* The purpose of the script is to convert all recency indices to breach ids for `breach_resolution`
9	* For backwards compatibility, all converted `breach_resolution` fields will have a boolean
10	* `useBreachId: true/false`
11	*/
12
13	import Knex from 'knex'
14	import knexConfig from '../../db/knexfile.js'
15	import { getAllBreachesFromDb } from '../../utils/hibp.js'
16	import { getAllEmailsAndBreaches } from '../../utils/breaches.js'
17	const knex = Knex(knexConfig)	×
18
19	const LIMIT = 1000 // with millions of records, we have to load a few at a time	×
20	let subscribersArr = []	×
21
22	const selectAndLockResolutions = async () => {	×
23	const trx = await knex.transaction()	×
24	let subscribers = []	×
25	try {	×
26	subscribers = await knex.select('id', 'primary_email', 'breach_resolution')	×
27	.from('subscribers')
28	.whereNotNull('breach_resolution')
29	.whereNull('db_migration_2')
30	.limit(LIMIT)
31	.orderBy('updated_at', 'desc')
32	.transacting(trx)
33	.forUpdate()
34
35	// update the lock
36	await Promise.all(subscribers.map(sub => {	×
37	const { id } = sub	×
38	return knex('subscribers')	×
39	.where('id', id)
40	.update({
41	db_migration_2: true
42	})
43	.transacting(trx)
44	}))
45
46	await trx.commit()	×
47	} catch (error) {
48	await trx.rollback()	×
49	console.error('select & mark rows failed!! first row:')	×
50	console.log({ first: subscribers[0] })	×
51	console.error(error)	×
52	}
53
54	return subscribers	×
55	}
56
57	/**
58	* Batch update
59	*
60	* @param {*} updateCollection
61	*/
62	const batchUpdate = async (updateCollection) => {	×
63	const trx = await knex.transaction()	×
64	try {	×
65	await Promise.all(updateCollection.map(tuple => {	×
66	const { user, updatedBreachesResolution } = tuple	×
67	return knex('subscribers')	×
68	.where('id', user.id)
69	.update({
70	breach_resolution: updatedBreachesResolution
71	})
72	.transacting(trx)
73	}))
74	await trx.commit()	×
75	} catch (error) {
76	await trx.rollback()	×
77	console.error('batch update failed!!')	×
78	console.log({ updateCollection })	×
79	console.error(error)	×
80	}
81	}
82
83	// Script begins here
84	const startTime = Date.now()	×
85	console.log(`Start time is: ${startTime}`)	×
86
87	// load all breaches for ref
88	const allBreaches = await getAllBreachesFromDb()	×
89	if (allBreaches && allBreaches.length > 0) console.log('breaches loaded successfully! ', allBreaches.length)	×
90	// console.log(JSON.stringify(allBreaches[0]))
91
92	// find all subscribers who resolved any breaches in the past,
93	// replace recency index with breach id
94
95	let failedToSelect = true	×
96	while (failedToSelect) {	×
97	try {	×
98	subscribersArr = await selectAndLockResolutions()	×
99	failedToSelect = false	×
100	} catch (e) {
101	console.error(e)	×
102	}
103	}
104
105	console.log(`Loaded # of subscribers: ${subscribersArr.length}`)	×
106	const updateCollection = []	×
107
108	for (const subscriber of subscribersArr) {	×
109	const { breach_resolution: v2 } = subscriber	×
110	// console.debug({ v2 })
111
112	// if useBreachId is set, skip because this breach_resolution has already been worked on
113	if (v2.useBreachId) {	×
114	console.log('Skipping since `useBreachId` is set already, this breach resolution is already converted')	×
115	continue	×
116	}
117
118	const newResolutions = {}	×
119
120	// fetch subscriber all breaches / email
121	let subscriberBreachesEmail
122	try {	×
123	subscriberBreachesEmail = await getAllEmailsAndBreaches(subscriber, allBreaches)	×
124	} catch (e) {
125	console.error('Cannot fetch subscriber breaches at the moment: ', e)	×
126	continue	×
127	}
128	// console.debug(JSON.stringify(subscriberBreachesEmail.verifiedEmails))
129
130	for (const email in v2) {	×
131	// console.debug({ email })
132	const resolutions = v2[email]	×
133	// console.debug({ resolutions })
134	newResolutions[email] = {}	×
135
136	for (const recencyIndex in resolutions) {	×
137	// console.debug({ recencyIndex })
138
139	// find subscriber's relevant recency index breach information
140	const ve = subscriberBreachesEmail.verifiedEmails?.filter(ve => ve.email === email)[0] \|\| {}	×
141	const subBreach = ve.breaches?.filter(b => Number(b.recencyIndex) === Number(recencyIndex))[0] \|\| null	×
142	const breachName = subBreach?.Name	×
143	console.debug({ breachName })	×
144
145	// find breach id for the breach
146	const breachId = allBreaches.find(b => b.Name === breachName)?.Id	×
147	console.log({ breachId })	×
148	newResolutions[email][breachId] = v2[email][recencyIndex]	×
149	}
150	}
151
152	// check if v2 is changed, if so, upsert the new v2
153	newResolutions.useBreachId = true	×
154	// console.log(JSON.stringify(newResolutions))
155	updateCollection.push({ user: subscriber, updatedBreachesResolution: newResolutions })	×
156	}
157
158	await batchUpdate(updateCollection)	×
159
160	console.log('Reaching the end of the table')	×
161	const endTime = Date.now()	×
162	console.log(`End time is: ${endTime}`)	×
163	console.log('Diff is: ', endTime - startTime)	×
164	process.exit()	×

mozilla / blurts-server / #13037

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous