baf86cd7-034b-4234-952d-79219c9c0900

Build # baf86cd7-034b-4234-952d-79219c9c0900

Build Type

push

circleci

Committed by GitHub

Commit Message

Merge pull request #2838 from mozilla/MNTOR-1173-Settings-email-breaches-count

Run Details

282 of 1371 branches covered (20.57%)

Branch coverage included in aggregate %.

5 of 5 new or added lines in 3 files covered. (100.0%)

959 of 3720 relevant lines covered (25.78%)

4.19 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/src/utils/hibp.js

/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

import mozlog from './log.js'
import AppConstants from '../app-constants.js'
import { fluentError } from './fluent.js'
import { getAllBreaches, upsertBreaches } from '../db/tables/breaches.js'
const { HIBP_THROTTLE_MAX_TRIES, HIBP_THROTTLE_DELAY, HIBP_API_ROOT, HIBP_KANON_API_ROOT, HIBP_KANON_API_TOKEN } = AppConstants

// TODO: fix hardcode
const HIBP_USER_AGENT = 'monitor/1.0.0'
// When HIBP "re-names" a breach, it keeps its old 'Name' value but gets a new 'Title'
// We use 'Name' in Firefox (via Remote Settings), so we have to maintain our own mapping of re-named breaches.
const RENAMED_BREACHES = ['covve']
const RENAMED_BREACHES_MAP = {
  covve: 'db8151dd'
}
const log = mozlog('hibp')

function _addStandardOptions (options = {}) {
  const hibpOptions = {
    headers: {
      'User-Agent': HIBP_USER_AGENT
    }
  }
  return Object.assign(options, hibpOptions)
}

async function _throttledFetch (url, reqOptions, tryCount = 1) {
  try {
    const response = await fetch(url, reqOptions)
    if (response.ok) return await response.json()

    switch (response.status) {
      case 404:
        // 404 can mean "no results", return undefined response
        return undefined
      case 429:
        log.info('_throttledFetch', { err: 'Error 429, tryCount: ' + tryCount })
        if (tryCount >= HIBP_THROTTLE_MAX_TRIES) {
          throw fluentError('error-hibp-throttled')
        } else {
          tryCount++
          await new Promise(resolve => setTimeout(resolve, HIBP_THROTTLE_DELAY * tryCount))
          return await _throttledFetch(url, reqOptions, tryCount)
        }
      default:
        throw new Error(`bad response: ${response.status}`)
    }
  } catch (err) {
    log.error('_throttledFetch', { err })
    throw fluentError('error-hibp-connect')
  }
}

async function req (path, options = {}) {
  const url = `${HIBP_API_ROOT}${path}`
  const reqOptions = _addStandardOptions(options)
  return await _throttledFetch(url, reqOptions)
}

async function kAnonReq (path, options = {}) {
  // Construct HIBP url and standard headers
  const url = `${HIBP_KANON_API_ROOT}${path}?code=${encodeURIComponent(HIBP_KANON_API_TOKEN)}`
  const reqOptions = _addStandardOptions(options)
  return await _throttledFetch(url, reqOptions)
}

/**
 * Sanitize data classes
 * ie. "Email Addresses" -> "email-addresses"
 * @param {Array} dataClasses
 * @returns Array sanitized data classes array
 */
function formatDataClassesArray (dataClasses) {
  return dataClasses.map(dataClass =>
    dataClass.toLowerCase()
      .replace(/[^-a-z0-9]/g, '-')
      .replace(/-{2,}/g, '-')
      .replace(/(^-|-$)/g, '')
  )
}

/**
 * Get all breaches from the database table "breaches",
 * sanitize it, and return a javascript array
 * @returns formatted all breaches array
 */
async function getAllBreachesFromDb () {
  let dbBreaches = []
  try {
    dbBreaches = await getAllBreaches()
  } catch (e) {
    log.error('getAllBreachesFromDb', 'No breaches exist in the database: ' + e)
    return dbBreaches
  }

  // TODO: we can do some filtering here for the most commonly used fields
  // TODO: change field names to camel case
  return dbBreaches.map(breach => ({
    Id: breach.id,
    Name: breach.name,
    Title: breach.title,
    Domain: breach.domain,
    BreachDate: breach.breach_date,
    AddedDate: breach.added_date,
    ModifiedDate: breach.modified_date,
    PwnCount: breach.pwn_count,
    Description: breach.description,
    LogoPath: breach.logo_path,
    DataClasses: breach.data_classes,
    IsVerified: breach.is_verified,
    IsFabricated: breach.is_fabricated,
    IsSensitive: breach.is_sensitive,
    IsRetired: breach.is_retired,
    IsSpamList: breach.is_spam_list,
    IsMalware: breach.is_malware
  }))
}

async function loadBreachesIntoApp (app) {
  try {
    // attempt to fetch breaches from the "breaches" database table
    const breaches = await getAllBreachesFromDb()
    log.debug('loadBreachesIntoApp', `loaded breaches from database: ${breaches.length}`)

    // if "breaches" table does not return results, fall back to HIBP request
    if (breaches?.length < 1) {
      const breachesResponse = await req('/breaches')
      log.debug('loadBreachesIntoApp', `loaded breaches from HIBP: ${breachesResponse.length}`)

      for (const breach of breachesResponse) {
        breach.DataClasses = formatDataClassesArray(breach.DataClasses)
        breach.LogoPath = /[^/]*$/.exec(breach.LogoPath)[0]
        breaches.push(breach)
      }

      // sync the "breaches" table with the latest from HIBP
      await upsertBreaches(breaches)
    }
    app.locals.breaches = breaches
    app.locals.breachesLoadedDateTime = Date.now()
  } catch (error) {
    throw fluentError('error-hibp-load-breaches')
  }
  log.info('done-loading-breaches', 'great success 👍')
}

/**
 * Get addresses and language from either subscribers or email_addresses fields:
 * @param {*} recipient
 * @returns
 */
function getAddressesAndLanguageForEmail (recipient) {
  const {
    all_emails_to_primary: allEmailsToPrimary,
    email: breachedEmail,
    primary_email: primaryEmail,
    signup_language: signupLanguage
  } = recipient

  if (breachedEmail) {
    return {
      breachedEmail,
      recipientEmail: allEmailsToPrimary ? primaryEmail : breachedEmail,
      signupLanguage
    }
  }

  return {
    breachedEmail: primaryEmail,
    recipientEmail: primaryEmail,
    signupLanguage
  }
}

/**
 * Filter breaches that we would not like to show.
 *
 * @param {Array} breaches
 * @returns {Array} filteredBreaches
 */
function getFilteredBreaches (breaches) {
  return breaches.filter(breach => (
    !breach.IsRetired &&
    !breach.IsSpamList &&
    !breach.IsFabricated &&
    breach.IsVerified &&
    breach.Domain !== ''
  ))
}

/**
A range of hashes can be searched by passing the hash prefix in a GET request:
GET /breachedaccount/range/[hash prefix]

 * @param {string} sha1 first 6 chars of email sha1
 * @param {Array} allBreaches
 * @param {Boolean} includeSensitive
 * @param {Boolean} filterBreaches
 * @returns
 */
async function getBreachesForEmail (sha1, allBreaches, includeSensitive = false, filterBreaches = true) {
  let foundBreaches = []
  const sha1Prefix = sha1.slice(0, 6).toUpperCase()
  const path = `/breachedaccount/range/${sha1Prefix}`

  const response = await kAnonReq(path)
  if (!response) {
    return []
  }
  // Parse response body, format:
  // [
  //   {"hashSuffix":<suffix>,"websites":[<breach1Name>,...]},
  //   {"hashSuffix":<suffix>,"websites":[<breach1Name>,...]},
  // ]
  for (const breachedAccount of response) {
    if (sha1.toUpperCase() === sha1Prefix + breachedAccount.hashSuffix) {
      foundBreaches = allBreaches.filter(breach => breachedAccount.websites.includes(breach.Name))
      if (filterBreaches) {
        foundBreaches = getFilteredBreaches(foundBreaches)
      }

      // NOTE: DO NOT CHANGE THIS SORT LOGIC
      // We store breach resolutions by recency indices,
      // so that our DB does not contain any part of any user's list of accounts
      foundBreaches.sort((a, b) => {
        return new Date(b.AddedDate) - new Date(a.AddedDate)
      })

      break
    }
  }

  if (includeSensitive) {
    return foundBreaches
  }
  return foundBreaches.filter(
    breach => !breach.IsSensitive
  )
}

function getBreachByName (allBreaches, breachName) {
  breachName = breachName.toLowerCase()
  if (RENAMED_BREACHES.includes(breachName)) {
    breachName = RENAMED_BREACHES_MAP[breachName]
  }
  const foundBreach = allBreaches.find(breach => breach.Name.toLowerCase() === breachName)
  return foundBreach
}

/**
 * A range can be subscribed for callbacks with the following request:
 * POST /range/subscribe
 * {
 *   hashPrefix:"[hash prefix]"
 * }
 * There are two possible response codes that can be returned:
 * 1. HTTP 201: New range subscription has been created
 * 2. HTTP 200: Range subscription already exists
 * @param {string} sha1 sha1 of the email being subscribed
 * @returns 200 or 201 response codes
 */
async function subscribeHash (sha1) {
  const sha1Prefix = sha1.slice(0, 6).toUpperCase()
  const path = '/range/subscribe'
  const options = {
    Method: 'POST',
    Body: { hashPrefix: sha1Prefix }
  }

  return await kAnonReq(path, options)
}

/**
 * A range subscription can be deleted with the following request:
 * DELETE /range/[hash prefix]

 * There is one possible response code that can be returned:
 * HTTP 200: Range subscription successfully deleted

 * @param {string} sha1 sha1 of the email being subscribed
 * @returns 200 response codes
 */
async function deleteSubscribedHash (sha1) {
  const sha1Prefix = sha1.slice(0, 6).toUpperCase()
  const path = `/range/${sha1Prefix}`
  const options = {
    Method: 'DELETE'
  }

  return await kAnonReq(path, options)
}

export {
  req,
  kAnonReq,
  formatDataClassesArray,
  loadBreachesIntoApp,
  getAddressesAndLanguageForEmail,
  getBreachesForEmail,
  getBreachByName,
  getAllBreachesFromDb,
  getFilteredBreaches,
  subscribeHash,
  deleteSubscribedHash
}

1	/* This Source Code Form is subject to the terms of the Mozilla Public
2	* License, v. 2.0. If a copy of the MPL was not distributed with this
3	* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
4
5	import mozlog from './log.js'
6	import AppConstants from '../app-constants.js'
7	import { fluentError } from './fluent.js'
8	import { getAllBreaches, upsertBreaches } from '../db/tables/breaches.js'
9	const { HIBP_THROTTLE_MAX_TRIES, HIBP_THROTTLE_DELAY, HIBP_API_ROOT, HIBP_KANON_API_ROOT, HIBP_KANON_API_TOKEN } = AppConstants	×
10
11	// TODO: fix hardcode
12	const HIBP_USER_AGENT = 'monitor/1.0.0'	×
13	// When HIBP "re-names" a breach, it keeps its old 'Name' value but gets a new 'Title'
14	// We use 'Name' in Firefox (via Remote Settings), so we have to maintain our own mapping of re-named breaches.
15	const RENAMED_BREACHES = ['covve']	×
16	const RENAMED_BREACHES_MAP = {	×
17	covve: 'db8151dd'
18	}
19	const log = mozlog('hibp')	×
20
21	function _addStandardOptions (options = {}) {	×
22	const hibpOptions = {	×
23	headers: {
24	'User-Agent': HIBP_USER_AGENT
25	}
26	}
27	return Object.assign(options, hibpOptions)	×
28	}
29
30	async function _throttledFetch (url, reqOptions, tryCount = 1) {	×
31	try {	×
32	const response = await fetch(url, reqOptions)	×
33	if (response.ok) return await response.json()	×
34
35	switch (response.status) {	×
36	case 404:
37	// 404 can mean "no results", return undefined response
38	return undefined	×
39	case 429:
40	log.info('_throttledFetch', { err: 'Error 429, tryCount: ' + tryCount })	×
41	if (tryCount >= HIBP_THROTTLE_MAX_TRIES) {	×
42	throw fluentError('error-hibp-throttled')	×
43	} else {
44	tryCount++	×
45	await new Promise(resolve => setTimeout(resolve, HIBP_THROTTLE_DELAY * tryCount))	×
46	return await _throttledFetch(url, reqOptions, tryCount)	×
47	}
48	default:
49	throw new Error(`bad response: ${response.status}`)	×
50	}
51	} catch (err) {
52	log.error('_throttledFetch', { err })	×
53	throw fluentError('error-hibp-connect')	×
54	}
55	}
56
57	async function req (path, options = {}) {	×
58	const url = `${HIBP_API_ROOT}${path}`	×
59	const reqOptions = _addStandardOptions(options)	×
60	return await _throttledFetch(url, reqOptions)	×
61	}
62
63	async function kAnonReq (path, options = {}) {	×
64	// Construct HIBP url and standard headers
65	const url = `${HIBP_KANON_API_ROOT}${path}?code=${encodeURIComponent(HIBP_KANON_API_TOKEN)}`	×
66	const reqOptions = _addStandardOptions(options)	×
67	return await _throttledFetch(url, reqOptions)	×
68	}
69
70	/**
71	* Sanitize data classes
72	* ie. "Email Addresses" -> "email-addresses"
73	* @param {Array} dataClasses
74	* @returns Array sanitized data classes array
75	*/
76	function formatDataClassesArray (dataClasses) {
77	return dataClasses.map(dataClass =>	×
78	dataClass.toLowerCase()	×
79	.replace(/[^-a-z0-9]/g, '-')
80	.replace(/-{2,}/g, '-')
81	.replace(/(^-\|-$)/g, '')
82	)
83	}
84
85	/**
86	* Get all breaches from the database table "breaches",
87	* sanitize it, and return a javascript array
88	* @returns formatted all breaches array
89	*/
90	async function getAllBreachesFromDb () {
91	let dbBreaches = []	×
92	try {	×
93	dbBreaches = await getAllBreaches()	×
94	} catch (e) {
95	log.error('getAllBreachesFromDb', 'No breaches exist in the database: ' + e)	×
96	return dbBreaches	×
97	}
98
99	// TODO: we can do some filtering here for the most commonly used fields
100	// TODO: change field names to camel case
101	return dbBreaches.map(breach => ({	×
102	Id: breach.id,
103	Name: breach.name,
104	Title: breach.title,
105	Domain: breach.domain,
106	BreachDate: breach.breach_date,
107	AddedDate: breach.added_date,
108	ModifiedDate: breach.modified_date,
109	PwnCount: breach.pwn_count,
110	Description: breach.description,
111	LogoPath: breach.logo_path,
112	DataClasses: breach.data_classes,
113	IsVerified: breach.is_verified,
114	IsFabricated: breach.is_fabricated,
115	IsSensitive: breach.is_sensitive,
116	IsRetired: breach.is_retired,
117	IsSpamList: breach.is_spam_list,
118	IsMalware: breach.is_malware
119	}))
120	}
121
122	async function loadBreachesIntoApp (app) {
123	try {	×
124	// attempt to fetch breaches from the "breaches" database table
125	const breaches = await getAllBreachesFromDb()	×
126	log.debug('loadBreachesIntoApp', `loaded breaches from database: ${breaches.length}`)	×
127
128	// if "breaches" table does not return results, fall back to HIBP request
129	if (breaches?.length < 1) {	×
130	const breachesResponse = await req('/breaches')	×
131	log.debug('loadBreachesIntoApp', `loaded breaches from HIBP: ${breachesResponse.length}`)	×
132
133	for (const breach of breachesResponse) {	×
134	breach.DataClasses = formatDataClassesArray(breach.DataClasses)	×
135	breach.LogoPath = /[^/]*$/.exec(breach.LogoPath)[0]	×
136	breaches.push(breach)	×
137	}
138
139	// sync the "breaches" table with the latest from HIBP
140	await upsertBreaches(breaches)	×
141	}
142	app.locals.breaches = breaches	×
143	app.locals.breachesLoadedDateTime = Date.now()	×
144	} catch (error) {
145	throw fluentError('error-hibp-load-breaches')	×
146	}
147	log.info('done-loading-breaches', 'great success 👍')	×
148	}
149
150	/**
151	* Get addresses and language from either subscribers or email_addresses fields:
152	* @param {*} recipient
153	* @returns
154	*/
155	function getAddressesAndLanguageForEmail (recipient) {
156	const {
157	all_emails_to_primary: allEmailsToPrimary,
158	email: breachedEmail,
159	primary_email: primaryEmail,
160	signup_language: signupLanguage
161	} = recipient	×
162
163	if (breachedEmail) {	×
164	return {	×
165	breachedEmail,
166	recipientEmail: allEmailsToPrimary ? primaryEmail : breachedEmail,	×
167	signupLanguage
168	}
169	}
170
171	return {	×
172	breachedEmail: primaryEmail,
173	recipientEmail: primaryEmail,
174	signupLanguage
175	}
176	}
177
178	/**
179	* Filter breaches that we would not like to show.
180	*
181	* @param {Array} breaches
182	* @returns {Array} filteredBreaches
183	*/
184	function getFilteredBreaches (breaches) {
185	return breaches.filter(breach => (	×
186	!breach.IsRetired &&	×
187	!breach.IsSpamList &&
188	!breach.IsFabricated &&
189	breach.IsVerified &&
190	breach.Domain !== ''
191	))
192	}
193
194	/**
195	A range of hashes can be searched by passing the hash prefix in a GET request:
196	GET /breachedaccount/range/[hash prefix]
197
198	* @param {string} sha1 first 6 chars of email sha1
199	* @param {Array} allBreaches
200	* @param {Boolean} includeSensitive
201	* @param {Boolean} filterBreaches
202	* @returns
203	*/
204	async function getBreachesForEmail (sha1, allBreaches, includeSensitive = false, filterBreaches = true) {	×
205	let foundBreaches = []	×
206	const sha1Prefix = sha1.slice(0, 6).toUpperCase()	×
207	const path = `/breachedaccount/range/${sha1Prefix}`	×
208
209	const response = await kAnonReq(path)	×
210	if (!response) {	×
211	return []	×
212	}
213	// Parse response body, format:
214	// [
215	// {"hashSuffix":<suffix>,"websites":[<breach1Name>,...]},
216	// {"hashSuffix":<suffix>,"websites":[<breach1Name>,...]},
217	// ]
218	for (const breachedAccount of response) {	×
219	if (sha1.toUpperCase() === sha1Prefix + breachedAccount.hashSuffix) {	×
220	foundBreaches = allBreaches.filter(breach => breachedAccount.websites.includes(breach.Name))	×
221	if (filterBreaches) {	×
222	foundBreaches = getFilteredBreaches(foundBreaches)	×
223	}
224
225	// NOTE: DO NOT CHANGE THIS SORT LOGIC
226	// We store breach resolutions by recency indices,
227	// so that our DB does not contain any part of any user's list of accounts
228	foundBreaches.sort((a, b) => {	×
229	return new Date(b.AddedDate) - new Date(a.AddedDate)	×
230	})
231
232	break	×
233	}
234	}
235
236	if (includeSensitive) {	×
237	return foundBreaches	×
238	}
239	return foundBreaches.filter(	×
240	breach => !breach.IsSensitive	×
241	)
242	}
243
244	function getBreachByName (allBreaches, breachName) {
245	breachName = breachName.toLowerCase()	×
246	if (RENAMED_BREACHES.includes(breachName)) {	×
247	breachName = RENAMED_BREACHES_MAP[breachName]	×
248	}
249	const foundBreach = allBreaches.find(breach => breach.Name.toLowerCase() === breachName)	×
250	return foundBreach	×
251	}
252
253	/**
254	* A range can be subscribed for callbacks with the following request:
255	* POST /range/subscribe
256	* {
257	* hashPrefix:"[hash prefix]"
258	* }
259	* There are two possible response codes that can be returned:
260	* 1. HTTP 201: New range subscription has been created
261	* 2. HTTP 200: Range subscription already exists
262	* @param {string} sha1 sha1 of the email being subscribed
263	* @returns 200 or 201 response codes
264	*/
265	async function subscribeHash (sha1) {
266	const sha1Prefix = sha1.slice(0, 6).toUpperCase()	×
267	const path = '/range/subscribe'	×
268	const options = {	×
269	Method: 'POST',
270	Body: { hashPrefix: sha1Prefix }
271	}
272
273	return await kAnonReq(path, options)	×
274	}
275
276	/**
277	* A range subscription can be deleted with the following request:
278	* DELETE /range/[hash prefix]
279
280	* There is one possible response code that can be returned:
281	* HTTP 200: Range subscription successfully deleted
282
283	* @param {string} sha1 sha1 of the email being subscribed
284	* @returns 200 response codes
285	*/
286	async function deleteSubscribedHash (sha1) {
287	const sha1Prefix = sha1.slice(0, 6).toUpperCase()	×
288	const path = `/range/${sha1Prefix}`	×
289	const options = {	×
290	Method: 'DELETE'
291	}
292
293	return await kAnonReq(path, options)	×
294	}
295
296	export {
297	req,
298	kAnonReq,
299	formatDataClassesArray,
300	loadBreachesIntoApp,
301	getAddressesAndLanguageForEmail,
302	getBreachesForEmail,
303	getBreachByName,
304	getAllBreachesFromDb,
305	getFilteredBreaches,
306	subscribeHash,
307	deleteSubscribedHash
308	}

mozilla / blurts-server / baf86cd7-034b-4234-952d-79219c9c0900

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous