• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

mozilla / blurts-server / #12893

pending completion
#12893

push

circleci

web-flow
Merge pull request #2923 from mozilla/MNTOR-1395

MNTOR-1395: kube job convert breach resolution script

282 of 1527 branches covered (18.47%)

Branch coverage included in aggregate %.

73 of 73 new or added lines in 2 files covered. (100.0%)

959 of 4139 relevant lines covered (23.17%)

1.93 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0
/src/scripts/prod-experiments/benchmarkWithUpsert.js
1
/* This Source Code Form is subject to the terms of the Mozilla Public
2
 * License, v. 2.0. If a copy of the MPL was not distributed with this
3
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
4

5
/**
6
 * Executes once
7
 * The purpose of the script is to convert all `subscriber.breaches_resolved` to `subscriber.breaches_resolution`
8
 * with the goal of deprecating the column
9
 */
10

11
import Knex from 'knex'
12
import knexConfig from '../../db/knexfile.js'
13
import { getAllBreachesFromDb } from '../../utils/hibp.js'
14
import { getAllEmailsAndBreaches } from '../../utils/breaches.js'
15
import { BreachDataTypes } from '../../utils/breach-resolution.js'
16
const knex = Knex(knexConfig)
×
17

18
const LIMIT = 1000 // with millions of records, we have to load a few at a time
×
19
let CAP = 5000 // cap the experiment
×
20
if (process.argv.length > 2) {
×
21
  CAP = process.argv[2]
×
22
  console.log('using cap passed in: ', CAP)
×
23
}
24
let offset = 0 // looping through all records with offset
×
25
let subscribersArr = []
×
26

27
/**
28
 * Batch update
29
 *
30
 * @param {*} updateCollection
31
 */
32
const batchUpdate = async (updateCollection) => {
×
33
  const trx = await knex.transaction()
×
34
  try {
×
35
    await Promise.all(updateCollection.map(tuple => {
×
36
      const { user, updatedBreachesResolution } = tuple
×
37
      return knex('subscribers')
×
38
        .where('id', user.id)
39
        .update({
40
          breach_resolution: updatedBreachesResolution
41
        })
42
        .transacting(trx)
43
    })
44
    )
45
    await trx.commit()
×
46
  } catch (error) {
47
    await trx.rollback()
×
48
    console.error('batch update failed!!')
×
49
    console.log({ updateCollection })
×
50
    console.error(error)
×
51
  }
52
}
53

54
const startTime = Date.now()
×
55
console.log(`Start time is: ${startTime}`)
×
56

57
// load all breaches for ref
58
const allBreaches = await getAllBreachesFromDb()
×
59
if (allBreaches && allBreaches.length > 0) console.log('breaches loaded successfully! ', allBreaches.length)
×
60

61
// find all subscribers who resolved any breaches in the past, convert those
62
// records into the new v2 format
63
do {
×
64
  console.log(`Converting breaches_resolved to breach_resolution - start: ${offset} limit: ${LIMIT}`)
×
65
  subscribersArr = await knex
×
66
    .select('id', 'primary_email', 'breaches_resolved', 'breach_resolution')
67
    .from('subscribers')
68
    .whereNotNull('breaches_resolved')
69
    .limit(LIMIT)
70
    .offset(offset)
71
    .orderBy('updated_at', 'desc')
72

73
  console.log(`Loaded # of subscribers: ${subscribersArr.length}`)
×
74
  const updateCollection = []
×
75

76
  for (const subscriber of subscribersArr) {
×
77
    let { breaches_resolved: v1, breach_resolution: v2 } = subscriber
×
78
    let isV2Changed = false // use a boolean to track if v2 has been changed, only upsert if so
×
79

80
    // fetch subscriber all breaches / email
81
    const subscriberBreachesEmail = await getAllEmailsAndBreaches(subscriber, allBreaches)
×
82
    // console.debug(JSON.stringify(subscriberBreachesEmail.verifiedEmails))
83

84
    for (const [email, resolvedRecencyIndices] of Object.entries(v1)) {
×
85
      // console.debug({ email })
86
      // console.debug({ resolvedRecencyIndices })
87
      for (const recencyIndex of resolvedRecencyIndices) {
×
88
        // console.debug({ recencyIndex })
89
        // find subscriber's relevant recency index breach information
90
        const ve = subscriberBreachesEmail.verifiedEmails?.filter(e => e.email === email)[0] || {}
×
91
        // console.debug({ ve })
92
        const subBreach = ve.breaches?.filter(b => Number(b.recencyIndex) === Number(recencyIndex))[0] || null
×
93
        // console.debug({ subBreach })
94

95
        if (!subBreach || !subBreach.DataClasses) {
×
96
          console.warn(`SKIP: Cannot find subscribers breach and data types - recency: ${recencyIndex} email: ${email}`)
×
97
          continue
×
98
        }
99

100
        // if email does not exist in v2, we need to add it to the object
101
        // format: {email: { recencyIndex: { isResolved: true, resolutionsChecked: [DataTypes]}}}
102
        if (!v2) v2 = {}
×
103
        if (!v2[email]) {
×
104
          v2[email] = {
×
105
            [recencyIndex]: {
106
              isResolved: true,
107
              resolutionsChecked: subBreach?.DataClasses || [BreachDataTypes.General]
×
108
            }
109
          }
110

111
          isV2Changed = true
×
112
        }
113
        if (v2[email][recencyIndex]?.isResolved) {
×
114
          console.log(`recencyIndex ${recencyIndex} exists in v2 and is resolved, no changes`)
×
115
        } else {
116
          console.log(`recencyIndex ${recencyIndex} either does not exist or is not resolved, overwriting`)
×
117
          v2[email][recencyIndex] = {
×
118
            isResolved: true,
119
            resolutionsChecked: subBreach?.DataClasses
120
          }
121
          isV2Changed = true
×
122
        }
123
      }
124
    }
125

126
    // check if v2 is changed, if so, upsert the new v2
127
    if (isV2Changed) {
×
128
      console.log('upsert for subscriber: ', subscriber.primary_email)
×
129
      updateCollection.push({ user: subscriber, updatedBreachesResolution: v2 })
×
130
    }
131
  }
132
  await batchUpdate(updateCollection)
×
133
  offset += LIMIT
×
134
} while (subscribersArr.length === LIMIT && offset <= CAP)
×
135

136
// breaking out of do..while loop
137
console.log('Reaching the end of the table, offset ended at', offset)
×
138
const endTime = Date.now()
×
139
console.log(`End time is: ${endTime}`)
×
140
console.log('Diff is: ', endTime - startTime)
×
141
process.exit()
×
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2025 Coveralls, Inc