aeb4115e-9e94-40de-bfe5-29c2b3acea36

Build # aeb4115e-9e94-40de-bfe5-29c2b3acea36

Build Type

push

circleci

Committed by GitHub

Commit Message

Merge pull request #2728 from mozilla/MNTOR-1044/replace-GOT

Run Details

278 of 1070 branches covered (25.98%)

Branch coverage included in aggregate %.

31 of 31 new or added lines in 3 files covered. (100.0%)

948 of 2889 relevant lines covered (32.81%)

5.19 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/src/utils/hibp.js

import mozlog from './log.js'
import AppConstants from '../app-constants.js'
import { fluentError } from './fluent.js'
const { HIBP_THROTTLE_MAX_TRIES, HIBP_THROTTLE_DELAY, HIBP_API_ROOT, HIBP_KANON_API_ROOT, HIBP_KANON_API_TOKEN } = AppConstants

// TODO: fix hardcode
const HIBP_USER_AGENT = 'monitor/1.0.0'
// When HIBP "re-names" a breach, it keeps its old 'Name' value but gets a new 'Title'
// We use 'Name' in Firefox (via Remote Settings), so we have to maintain our own mapping of re-named breaches.
const RENAMED_BREACHES = ['covve']
const RENAMED_BREACHES_MAP = {
  covve: 'db8151dd'
}
const log = mozlog('hibp')

function _addStandardOptions (options = {}) {
  const hibpOptions = {
    headers: {
      'User-Agent': HIBP_USER_AGENT
    }
  }
  return Object.assign(options, hibpOptions)
}

async function _throttledFetch (url, reqOptions, tryCount = 1) {
  try {
    const response = await fetch(url, reqOptions)
    if (response.ok) return await response.json()

    switch (response.status) {
      case 404:
        // 404 can mean "no results", return undefined response
        return undefined
      case 429:
        log.info('_throttledFetch', { err: 'Error 429, tryCount: ' + tryCount })
        if (tryCount >= HIBP_THROTTLE_MAX_TRIES) {
          throw fluentError('error-hibp-throttled')
        } else {
          tryCount++
          await new Promise(resolve => setTimeout(resolve, HIBP_THROTTLE_DELAY * tryCount))
          return await _throttledFetch(url, reqOptions, tryCount)
        }
      default:
        throw new Error(`bad response: ${response.status}`)
    }
  } catch (err) {
    log.error('_throttledFetch', { err })
    throw fluentError('error-hibp-connect')
  }
}

async function req (path, options = {}) {
  const url = `${HIBP_API_ROOT}${path}`
  const reqOptions = _addStandardOptions(options)
  return await _throttledFetch(url, reqOptions)
}

async function kAnonReq (path, options = {}) {
  // Construct HIBP url and standard headers
  const url = `${HIBP_KANON_API_ROOT}${path}?code=${encodeURIComponent(HIBP_KANON_API_TOKEN)}`
  const reqOptions = _addStandardOptions(options)
  return await _throttledFetch(url, reqOptions)
}

function matchFluentID (dataCategory) {
  return dataCategory.toLowerCase()
    .replace(/[^-a-z0-9]/g, '-')
    .replace(/-{2,}/g, '-')
    .replace(/(^-|-$)/g, '')
}

function formatDataClassesArray (dataCategories) {
  const formattedArray = []
  dataCategories.forEach(category => {
    formattedArray.push(matchFluentID(category))
  })
  return formattedArray
}

async function loadBreachesIntoApp (app) {
  try {
    const breachesResponse = await req('/breaches')
    const breaches = []

    for (const breach of breachesResponse) {
      breach.DataClasses = formatDataClassesArray(breach.DataClasses)
      breach.LogoPath = /[^/]*$/.exec(breach.LogoPath)[0]
      breaches.push(breach)
    }
    app.locals.breaches = breaches
    app.locals.breachesLoadedDateTime = Date.now()
    app.locals.latestBreach = getLatestBreach(breaches)
    app.locals.mostRecentBreachDateTime = app.locals.latestBreach.AddedDate
  } catch (error) {
    throw fluentError('error-hibp-load-breaches')
  }
  log.info('done-loading-breaches', 'great success 👍')
}
/**
A range of hashes can be searched by passing the hash prefix in a GET request:
GET /breachedaccount/range/[hash prefix]

 * @param {string} sha1 first 6 chars of email sha1
 * @param {*} allBreaches
 * @param {*} includeSensitive
 * @param {*} filterBreaches
 * @returns
 */
async function getBreachesForEmail (sha1, allBreaches, includeSensitive = false, filterBreaches = true) {
  let foundBreaches = []
  const sha1Prefix = sha1.slice(0, 6).toUpperCase()
  const path = `/breachedaccount/range/${sha1Prefix}`

  const response = await kAnonReq(path)
  if (!response) {
    return []
  }
  // Parse response body, format:
  // [
  //   {"hashSuffix":<suffix>,"websites":[<breach1Name>,...]},
  //   {"hashSuffix":<suffix>,"websites":[<breach1Name>,...]},
  // ]
  for (const breachedAccount of response) {
    if (sha1.toUpperCase() === sha1Prefix + breachedAccount.hashSuffix) {
      foundBreaches = allBreaches.filter(breach => breachedAccount.websites.includes(breach.Name))
      if (filterBreaches) {
        foundBreaches = filterBreaches(foundBreaches)
      }

      // NOTE: DO NOT CHANGE THIS SORT LOGIC
      // We store breach resolutions by recency indices,
      // so that our DB does not contain any part of any user's list of accounts
      foundBreaches.sort((a, b) => {
        return new Date(b.AddedDate) - new Date(a.AddedDate)
      })

      break
    }
  }

  if (includeSensitive) {
    return foundBreaches
  }
  return foundBreaches.filter(
    breach => !breach.IsSensitive
  )
}

function getBreachByName (allBreaches, breachName) {
  breachName = breachName.toLowerCase()
  if (RENAMED_BREACHES.includes(breachName)) {
    breachName = RENAMED_BREACHES_MAP[breachName]
  }
  const foundBreach = allBreaches.find(breach => breach.Name.toLowerCase() === breachName)
  return foundBreach
}

function filterBreaches (breaches) {
  return breaches.filter(
    breach => !breach.IsRetired &&
                !breach.IsSpamList &&
                !breach.IsFabricated &&
                breach.IsVerified &&
                breach.Domain !== ''
  )
}

function getLatestBreach (breaches) {
  let latestBreach = {}
  let latestBreachDateTime = new Date(0)
  for (const breach of breaches) {
    if (breach.IsSensitive) {
      continue
    }
    const breachAddedDate = new Date(breach.AddedDate)
    if (breachAddedDate > latestBreachDateTime) {
      latestBreachDateTime = breachAddedDate
      latestBreach = breach
    }
  }
  return latestBreach
}
/**
A range can be subscribed for callbacks with the following request:
POST /range/subscribe
{
  hashPrefix:"[hash prefix]"
}
There are two possible response codes that will be returned:
1. HTTP 201: New range subscription has been created
2. HTTP 200: Range subscription already exists
 * @param {string} sha1 first 6 chars of sha1 of the email being subscribed
 * @returns 200 or 201 response codes
 */
async function subscribeHash (sha1) {
  const sha1Prefix = sha1.slice(0, 6).toUpperCase()
  const path = '/range/subscribe'
  const options = {
    method: 'POST',
    json: { hashPrefix: sha1Prefix }
  }

  return await kAnonReq(path, options)
}

export {
  req,
  kAnonReq,
  matchFluentID,
  formatDataClassesArray,
  loadBreachesIntoApp,
  getBreachesForEmail,
  getBreachByName,
  filterBreaches,
  getLatestBreach,
  subscribeHash
}

1	import mozlog from './log.js'
2	import AppConstants from '../app-constants.js'
3	import { fluentError } from './fluent.js'
4	const { HIBP_THROTTLE_MAX_TRIES, HIBP_THROTTLE_DELAY, HIBP_API_ROOT, HIBP_KANON_API_ROOT, HIBP_KANON_API_TOKEN } = AppConstants	×
5
6	// TODO: fix hardcode
7	const HIBP_USER_AGENT = 'monitor/1.0.0'	×
8	// When HIBP "re-names" a breach, it keeps its old 'Name' value but gets a new 'Title'
9	// We use 'Name' in Firefox (via Remote Settings), so we have to maintain our own mapping of re-named breaches.
10	const RENAMED_BREACHES = ['covve']	×
11	const RENAMED_BREACHES_MAP = {	×
12	covve: 'db8151dd'
13	}
14	const log = mozlog('hibp')	×
15
16	function _addStandardOptions (options = {}) {	×
17	const hibpOptions = {	×
18	headers: {
19	'User-Agent': HIBP_USER_AGENT
20	}
21	}
22	return Object.assign(options, hibpOptions)	×
23	}
24
25	async function _throttledFetch (url, reqOptions, tryCount = 1) {	×
26	try {	×
27	const response = await fetch(url, reqOptions)	×
28	if (response.ok) return await response.json()	×
29
30	switch (response.status) {	×
31	case 404:
32	// 404 can mean "no results", return undefined response
33	return undefined	×
34	case 429:
35	log.info('_throttledFetch', { err: 'Error 429, tryCount: ' + tryCount })	×
36	if (tryCount >= HIBP_THROTTLE_MAX_TRIES) {	×
37	throw fluentError('error-hibp-throttled')	×
38	} else {
39	tryCount++	×
40	await new Promise(resolve => setTimeout(resolve, HIBP_THROTTLE_DELAY * tryCount))	×
41	return await _throttledFetch(url, reqOptions, tryCount)	×
42	}
43	default:
44	throw new Error(`bad response: ${response.status}`)	×
45	}
46	} catch (err) {
47	log.error('_throttledFetch', { err })	×
48	throw fluentError('error-hibp-connect')	×
49	}
50	}
51
52	async function req (path, options = {}) {	×
53	const url = `${HIBP_API_ROOT}${path}`	×
54	const reqOptions = _addStandardOptions(options)	×
55	return await _throttledFetch(url, reqOptions)	×
56	}
57
58	async function kAnonReq (path, options = {}) {	×
59	// Construct HIBP url and standard headers
60	const url = `${HIBP_KANON_API_ROOT}${path}?code=${encodeURIComponent(HIBP_KANON_API_TOKEN)}`	×
61	const reqOptions = _addStandardOptions(options)	×
62	return await _throttledFetch(url, reqOptions)	×
63	}
64
65	function matchFluentID (dataCategory) {
66	return dataCategory.toLowerCase()	×
67	.replace(/[^-a-z0-9]/g, '-')
68	.replace(/-{2,}/g, '-')
69	.replace(/(^-\|-$)/g, '')
70	}
71
72	function formatDataClassesArray (dataCategories) {
73	const formattedArray = []	×
74	dataCategories.forEach(category => {	×
75	formattedArray.push(matchFluentID(category))	×
76	})
77	return formattedArray	×
78	}
79
80	async function loadBreachesIntoApp (app) {
81	try {	×
82	const breachesResponse = await req('/breaches')	×
83	const breaches = []	×
84
85	for (const breach of breachesResponse) {	×
86	breach.DataClasses = formatDataClassesArray(breach.DataClasses)	×
87	breach.LogoPath = /[^/]*$/.exec(breach.LogoPath)[0]	×
88	breaches.push(breach)	×
89	}
90	app.locals.breaches = breaches	×
91	app.locals.breachesLoadedDateTime = Date.now()	×
92	app.locals.latestBreach = getLatestBreach(breaches)	×
93	app.locals.mostRecentBreachDateTime = app.locals.latestBreach.AddedDate	×
94	} catch (error) {
95	throw fluentError('error-hibp-load-breaches')	×
96	}
97	log.info('done-loading-breaches', 'great success 👍')	×
98	}
99	/**
100	A range of hashes can be searched by passing the hash prefix in a GET request:
101	GET /breachedaccount/range/[hash prefix]
102
103	* @param {string} sha1 first 6 chars of email sha1
104	* @param {*} allBreaches
105	* @param {*} includeSensitive
106	* @param {*} filterBreaches
107	* @returns
108	*/
109	async function getBreachesForEmail (sha1, allBreaches, includeSensitive = false, filterBreaches = true) {	×
110	let foundBreaches = []	×
111	const sha1Prefix = sha1.slice(0, 6).toUpperCase()	×
112	const path = `/breachedaccount/range/${sha1Prefix}`	×
113
114	const response = await kAnonReq(path)	×
115	if (!response) {	×
116	return []	×
117	}
118	// Parse response body, format:
119	// [
120	// {"hashSuffix":<suffix>,"websites":[<breach1Name>,...]},
121	// {"hashSuffix":<suffix>,"websites":[<breach1Name>,...]},
122	// ]
123	for (const breachedAccount of response) {	×
124	if (sha1.toUpperCase() === sha1Prefix + breachedAccount.hashSuffix) {	×
125	foundBreaches = allBreaches.filter(breach => breachedAccount.websites.includes(breach.Name))	×
126	if (filterBreaches) {	×
127	foundBreaches = filterBreaches(foundBreaches)	×
128	}
129
130	// NOTE: DO NOT CHANGE THIS SORT LOGIC
131	// We store breach resolutions by recency indices,
132	// so that our DB does not contain any part of any user's list of accounts
133	foundBreaches.sort((a, b) => {	×
134	return new Date(b.AddedDate) - new Date(a.AddedDate)	×
135	})
136
137	break	×
138	}
139	}
140
141	if (includeSensitive) {	×
142	return foundBreaches	×
143	}
144	return foundBreaches.filter(	×
145	breach => !breach.IsSensitive	×
146	)
147	}
148
149	function getBreachByName (allBreaches, breachName) {
150	breachName = breachName.toLowerCase()	×
151	if (RENAMED_BREACHES.includes(breachName)) {	×
152	breachName = RENAMED_BREACHES_MAP[breachName]	×
153	}
154	const foundBreach = allBreaches.find(breach => breach.Name.toLowerCase() === breachName)	×
155	return foundBreach	×
156	}
157
158	function filterBreaches (breaches) {
159	return breaches.filter(	×
160	breach => !breach.IsRetired &&	×
161	!breach.IsSpamList &&
162	!breach.IsFabricated &&
163	breach.IsVerified &&
164	breach.Domain !== ''
165	)
166	}
167
168	function getLatestBreach (breaches) {
169	let latestBreach = {}	×
170	let latestBreachDateTime = new Date(0)	×
171	for (const breach of breaches) {	×
172	if (breach.IsSensitive) {	×
173	continue	×
174	}
175	const breachAddedDate = new Date(breach.AddedDate)	×
176	if (breachAddedDate > latestBreachDateTime) {	×
177	latestBreachDateTime = breachAddedDate	×
178	latestBreach = breach	×
179	}
180	}
181	return latestBreach	×
182	}
183	/**
184	A range can be subscribed for callbacks with the following request:
185	POST /range/subscribe
186	{
187	hashPrefix:"[hash prefix]"
188	}
189	There are two possible response codes that will be returned:
190	1. HTTP 201: New range subscription has been created
191	2. HTTP 200: Range subscription already exists
192	* @param {string} sha1 first 6 chars of sha1 of the email being subscribed
193	* @returns 200 or 201 response codes
194	*/
195	async function subscribeHash (sha1) {
196	const sha1Prefix = sha1.slice(0, 6).toUpperCase()	×
197	const path = '/range/subscribe'	×
198	const options = {	×
199	method: 'POST',
200	json: { hashPrefix: sha1Prefix }
201	}
202
203	return await kAnonReq(path, options)	×
204	}
205
206	export {
207	req,
208	kAnonReq,
209	matchFluentID,
210	formatDataClassesArray,
211	loadBreachesIntoApp,
212	getBreachesForEmail,
213	getBreachByName,
214	filterBreaches,
215	getLatestBreach,
216	subscribeHash
217	}

mozilla / blurts-server / aeb4115e-9e94-40de-bfe5-29c2b3acea36

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous