#11668

pending completion

Build # #11668

Build Type

push

circleci

Committed by

web-flow

Commit Message

Merge pull request #2732 from mozilla/MNTOR-976

Mntor 976: Storage for Breaches

Run Details

278 of 1080 branches covered (25.74%)

Branch coverage included in aggregate %.

44 of 44 new or added lines in 3 files covered. (100.0%)

948 of 2953 relevant lines covered (32.1%)

2.57 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/src/utils/hibp.js

import mozlog from './log.js'
import AppConstants from '../app-constants.js'
import { fluentError } from './fluent.js'
import { getAllBreaches } from '../db/tables/breaches.js'
const { HIBP_THROTTLE_MAX_TRIES, HIBP_THROTTLE_DELAY, HIBP_API_ROOT, HIBP_KANON_API_ROOT, HIBP_KANON_API_TOKEN } = AppConstants

// TODO: fix hardcode
const HIBP_USER_AGENT = 'monitor/1.0.0'
// When HIBP "re-names" a breach, it keeps its old 'Name' value but gets a new 'Title'
// We use 'Name' in Firefox (via Remote Settings), so we have to maintain our own mapping of re-named breaches.
const RENAMED_BREACHES = ['covve']
const RENAMED_BREACHES_MAP = {
  covve: 'db8151dd'
}
const log = mozlog('hibp')

function _addStandardOptions (options = {}) {
  const hibpOptions = {
    headers: {
      'User-Agent': HIBP_USER_AGENT
    }
  }
  return Object.assign(options, hibpOptions)
}

async function _throttledFetch (url, reqOptions, tryCount = 1) {
  try {
    const response = await fetch(url, reqOptions)
    if (response.ok) return await response.json()

    switch (response.status) {
      case 404:
        // 404 can mean "no results", return undefined response
        return undefined
      case 429:
        log.info('_throttledFetch', { err: 'Error 429, tryCount: ' + tryCount })
        if (tryCount >= HIBP_THROTTLE_MAX_TRIES) {
          throw fluentError('error-hibp-throttled')
        } else {
          tryCount++
          await new Promise(resolve => setTimeout(resolve, HIBP_THROTTLE_DELAY * tryCount))
          return await _throttledFetch(url, reqOptions, tryCount)
        }
      default:
        throw new Error(`bad response: ${response.status}`)
    }
  } catch (err) {
    log.error('_throttledFetch', { err })
    throw fluentError('error-hibp-connect')
  }
}

async function req (path, options = {}) {
  const url = `${HIBP_API_ROOT}${path}`
  const reqOptions = _addStandardOptions(options)
  return await _throttledFetch(url, reqOptions)
}

async function kAnonReq (path, options = {}) {
  // Construct HIBP url and standard headers
  const url = `${HIBP_KANON_API_ROOT}${path}?code=${encodeURIComponent(HIBP_KANON_API_TOKEN)}`
  const reqOptions = _addStandardOptions(options)
  return await _throttledFetch(url, reqOptions)
}

function matchFluentID (dataCategory) {
  return dataCategory.toLowerCase()
    .replace(/[^-a-z0-9]/g, '-')
    .replace(/-{2,}/g, '-')
    .replace(/(^-|-$)/g, '')
}

function formatDataClassesArray (dataCategories) {
  const formattedArray = []
  dataCategories.forEach(category => {
    formattedArray.push(matchFluentID(category))
  })
  return formattedArray
}

/**
 * Get all breaches from the database table "breaches",
 * sanitize it, and return a javascript array
 * @returns formatted all breaches array
 */
async function getAllBreachesFromDb () {
  let dbBreaches = []
  try {
    dbBreaches = await getAllBreaches()
  } catch (e) {
    log.error('getAllBreachesFromDb', 'No breaches exist in the database: ' + e)
    return dbBreaches
  }

  // TODO: we can do some filtering here for the most commonly used fields
  // TODO: change field names to camel case
  return dbBreaches.map(breach => ({
    Name: breach.name,
    Title: breach.title,
    Domain: breach.domain,
    BreachDate: breach.breach_date,
    AddedDate: breach.added_date,
    ModifiedDate: breach.modified_date,
    PwnCount: breach.pwn_count,
    Description: breach.description,
    LogoPath: breach.logo_path,
    DataClasses: breach.data_classes,
    IsVerified: breach.is_verified,
    IsFabricated: breach.is_fabricated,
    IsSensitive: breach.is_sensitive,
    IsRetired: breach.is_retired,
    IsSpamList: breach.is_spam_list,
    IsMalware: breach.is_malware
  }))
}

async function loadBreachesIntoApp (app) {
  try {
    // attempt to fetch breaches from the "breaches" database table
    const breaches = await getAllBreachesFromDb()
    log.debug('loadBreachesIntoApp', `loaded breaches from database: ${breaches.length}`)

    // if "breaches" table does not return results, fall back to HIBP request
    if (breaches?.length < 1) {
      const breachesResponse = await req('/breaches')
      log.debug('loadBreachesIntoApp', `loaded breaches from HIBP: ${breachesResponse.length}`)

      for (const breach of breachesResponse) {
        breach.DataClasses = formatDataClassesArray(breach.DataClasses)
        breach.LogoPath = /[^/]*$/.exec(breach.LogoPath)[0]
        breaches.push(breach)
      }
    }
    app.locals.breaches = breaches
    app.locals.breachesLoadedDateTime = Date.now()
    app.locals.latestBreach = getLatestBreach(breaches)
    app.locals.mostRecentBreachDateTime = app.locals.latestBreach.AddedDate
  } catch (error) {
    throw fluentError('error-hibp-load-breaches')
  }
  log.info('done-loading-breaches', 'great success 👍')
}
/**
A range of hashes can be searched by passing the hash prefix in a GET request:
GET /breachedaccount/range/[hash prefix]

 * @param {string} sha1 first 6 chars of email sha1
 * @param {*} allBreaches
 * @param {*} includeSensitive
 * @param {*} filterBreaches
 * @returns
 */
async function getBreachesForEmail (sha1, allBreaches, includeSensitive = false, filterBreaches = true) {
  let foundBreaches = []
  const sha1Prefix = sha1.slice(0, 6).toUpperCase()
  const path = `/breachedaccount/range/${sha1Prefix}`

  const response = await kAnonReq(path)
  if (!response) {
    return []
  }
  // Parse response body, format:
  // [
  //   {"hashSuffix":<suffix>,"websites":[<breach1Name>,...]},
  //   {"hashSuffix":<suffix>,"websites":[<breach1Name>,...]},
  // ]
  for (const breachedAccount of response) {
    if (sha1.toUpperCase() === sha1Prefix + breachedAccount.hashSuffix) {
      foundBreaches = allBreaches.filter(breach => breachedAccount.websites.includes(breach.Name))
      if (filterBreaches) {
        foundBreaches = filterBreaches(foundBreaches)
      }

      // NOTE: DO NOT CHANGE THIS SORT LOGIC
      // We store breach resolutions by recency indices,
      // so that our DB does not contain any part of any user's list of accounts
      foundBreaches.sort((a, b) => {
        return new Date(b.AddedDate) - new Date(a.AddedDate)
      })

      break
    }
  }

  if (includeSensitive) {
    return foundBreaches
  }
  return foundBreaches.filter(
    breach => !breach.IsSensitive
  )
}

function getBreachByName (allBreaches, breachName) {
  breachName = breachName.toLowerCase()
  if (RENAMED_BREACHES.includes(breachName)) {
    breachName = RENAMED_BREACHES_MAP[breachName]
  }
  const foundBreach = allBreaches.find(breach => breach.Name.toLowerCase() === breachName)
  return foundBreach
}

function filterBreaches (breaches) {
  return breaches.filter(
    breach => !breach.IsRetired &&
                !breach.IsSpamList &&
                !breach.IsFabricated &&
                breach.IsVerified &&
                breach.Domain !== ''
  )
}

function getLatestBreach (breaches) {
  let latestBreach = {}
  let latestBreachDateTime = new Date(0)
  for (const breach of breaches) {
    if (breach.IsSensitive) {
      continue
    }
    const breachAddedDate = new Date(breach.AddedDate)
    if (breachAddedDate > latestBreachDateTime) {
      latestBreachDateTime = breachAddedDate
      latestBreach = breach
    }
  }
  return latestBreach
}
/**
A range can be subscribed for callbacks with the following request:
POST /range/subscribe
{
  hashPrefix:"[hash prefix]"
}
There are two possible response codes that will be returned:
1. HTTP 201: New range subscription has been created
2. HTTP 200: Range subscription already exists
 * @param {string} sha1 first 6 chars of sha1 of the email being subscribed
 * @returns 200 or 201 response codes
 */
async function subscribeHash (sha1) {
  const sha1Prefix = sha1.slice(0, 6).toUpperCase()
  const path = '/range/subscribe'
  const options = {
    method: 'POST',
    json: { hashPrefix: sha1Prefix }
  }

  return await kAnonReq(path, options)
}

export {
  req,
  kAnonReq,
  matchFluentID,
  formatDataClassesArray,
  loadBreachesIntoApp,
  getBreachesForEmail,
  getBreachByName,
  getAllBreachesFromDb,
  filterBreaches,
  getLatestBreach,
  subscribeHash
}

1	import mozlog from './log.js'
2	import AppConstants from '../app-constants.js'
3	import { fluentError } from './fluent.js'
4	import { getAllBreaches } from '../db/tables/breaches.js'
5	const { HIBP_THROTTLE_MAX_TRIES, HIBP_THROTTLE_DELAY, HIBP_API_ROOT, HIBP_KANON_API_ROOT, HIBP_KANON_API_TOKEN } = AppConstants	×
6
7	// TODO: fix hardcode
8	const HIBP_USER_AGENT = 'monitor/1.0.0'	×
9	// When HIBP "re-names" a breach, it keeps its old 'Name' value but gets a new 'Title'
10	// We use 'Name' in Firefox (via Remote Settings), so we have to maintain our own mapping of re-named breaches.
11	const RENAMED_BREACHES = ['covve']	×
12	const RENAMED_BREACHES_MAP = {	×
13	covve: 'db8151dd'
14	}
15	const log = mozlog('hibp')	×
16
17	function _addStandardOptions (options = {}) {	×
18	const hibpOptions = {	×
19	headers: {
20	'User-Agent': HIBP_USER_AGENT
21	}
22	}
23	return Object.assign(options, hibpOptions)	×
24	}
25
26	async function _throttledFetch (url, reqOptions, tryCount = 1) {	×
27	try {	×
28	const response = await fetch(url, reqOptions)	×
29	if (response.ok) return await response.json()	×
30
31	switch (response.status) {	×
32	case 404:
33	// 404 can mean "no results", return undefined response
34	return undefined	×
35	case 429:
36	log.info('_throttledFetch', { err: 'Error 429, tryCount: ' + tryCount })	×
37	if (tryCount >= HIBP_THROTTLE_MAX_TRIES) {	×
38	throw fluentError('error-hibp-throttled')	×
39	} else {
40	tryCount++	×
41	await new Promise(resolve => setTimeout(resolve, HIBP_THROTTLE_DELAY * tryCount))	×
42	return await _throttledFetch(url, reqOptions, tryCount)	×
43	}
44	default:
45	throw new Error(`bad response: ${response.status}`)	×
46	}
47	} catch (err) {
48	log.error('_throttledFetch', { err })	×
49	throw fluentError('error-hibp-connect')	×
50	}
51	}
52
53	async function req (path, options = {}) {	×
54	const url = `${HIBP_API_ROOT}${path}`	×
55	const reqOptions = _addStandardOptions(options)	×
56	return await _throttledFetch(url, reqOptions)	×
57	}
58
59	async function kAnonReq (path, options = {}) {	×
60	// Construct HIBP url and standard headers
61	const url = `${HIBP_KANON_API_ROOT}${path}?code=${encodeURIComponent(HIBP_KANON_API_TOKEN)}`	×
62	const reqOptions = _addStandardOptions(options)	×
63	return await _throttledFetch(url, reqOptions)	×
64	}
65
66	function matchFluentID (dataCategory) {
67	return dataCategory.toLowerCase()	×
68	.replace(/[^-a-z0-9]/g, '-')
69	.replace(/-{2,}/g, '-')
70	.replace(/(^-\|-$)/g, '')
71	}
72
73	function formatDataClassesArray (dataCategories) {
74	const formattedArray = []	×
75	dataCategories.forEach(category => {	×
76	formattedArray.push(matchFluentID(category))	×
77	})
78	return formattedArray	×
79	}
80
81	/**
82	* Get all breaches from the database table "breaches",
83	* sanitize it, and return a javascript array
84	* @returns formatted all breaches array
85	*/
86	async function getAllBreachesFromDb () {
87	let dbBreaches = []	×
88	try {	×
89	dbBreaches = await getAllBreaches()	×
90	} catch (e) {
91	log.error('getAllBreachesFromDb', 'No breaches exist in the database: ' + e)	×
92	return dbBreaches	×
93	}
94
95	// TODO: we can do some filtering here for the most commonly used fields
96	// TODO: change field names to camel case
97	return dbBreaches.map(breach => ({	×
98	Name: breach.name,
99	Title: breach.title,
100	Domain: breach.domain,
101	BreachDate: breach.breach_date,
102	AddedDate: breach.added_date,
103	ModifiedDate: breach.modified_date,
104	PwnCount: breach.pwn_count,
105	Description: breach.description,
106	LogoPath: breach.logo_path,
107	DataClasses: breach.data_classes,
108	IsVerified: breach.is_verified,
109	IsFabricated: breach.is_fabricated,
110	IsSensitive: breach.is_sensitive,
111	IsRetired: breach.is_retired,
112	IsSpamList: breach.is_spam_list,
113	IsMalware: breach.is_malware
114	}))
115	}
116
117	async function loadBreachesIntoApp (app) {
118	try {	×
119	// attempt to fetch breaches from the "breaches" database table
120	const breaches = await getAllBreachesFromDb()	×
121	log.debug('loadBreachesIntoApp', `loaded breaches from database: ${breaches.length}`)	×
122
123	// if "breaches" table does not return results, fall back to HIBP request
124	if (breaches?.length < 1) {	×
125	const breachesResponse = await req('/breaches')	×
126	log.debug('loadBreachesIntoApp', `loaded breaches from HIBP: ${breachesResponse.length}`)	×
127
128	for (const breach of breachesResponse) {	×
129	breach.DataClasses = formatDataClassesArray(breach.DataClasses)	×
130	breach.LogoPath = /[^/]*$/.exec(breach.LogoPath)[0]	×
131	breaches.push(breach)	×
132	}
133	}
134	app.locals.breaches = breaches	×
135	app.locals.breachesLoadedDateTime = Date.now()	×
136	app.locals.latestBreach = getLatestBreach(breaches)	×
137	app.locals.mostRecentBreachDateTime = app.locals.latestBreach.AddedDate	×
138	} catch (error) {
139	throw fluentError('error-hibp-load-breaches')	×
140	}
141	log.info('done-loading-breaches', 'great success 👍')	×
142	}
143	/**
144	A range of hashes can be searched by passing the hash prefix in a GET request:
145	GET /breachedaccount/range/[hash prefix]
146
147	* @param {string} sha1 first 6 chars of email sha1
148	* @param {*} allBreaches
149	* @param {*} includeSensitive
150	* @param {*} filterBreaches
151	* @returns
152	*/
153	async function getBreachesForEmail (sha1, allBreaches, includeSensitive = false, filterBreaches = true) {	×
154	let foundBreaches = []	×
155	const sha1Prefix = sha1.slice(0, 6).toUpperCase()	×
156	const path = `/breachedaccount/range/${sha1Prefix}`	×
157
158	const response = await kAnonReq(path)	×
159	if (!response) {	×
160	return []	×
161	}
162	// Parse response body, format:
163	// [
164	// {"hashSuffix":<suffix>,"websites":[<breach1Name>,...]},
165	// {"hashSuffix":<suffix>,"websites":[<breach1Name>,...]},
166	// ]
167	for (const breachedAccount of response) {	×
168	if (sha1.toUpperCase() === sha1Prefix + breachedAccount.hashSuffix) {	×
169	foundBreaches = allBreaches.filter(breach => breachedAccount.websites.includes(breach.Name))	×
170	if (filterBreaches) {	×
171	foundBreaches = filterBreaches(foundBreaches)	×
172	}
173
174	// NOTE: DO NOT CHANGE THIS SORT LOGIC
175	// We store breach resolutions by recency indices,
176	// so that our DB does not contain any part of any user's list of accounts
177	foundBreaches.sort((a, b) => {	×
178	return new Date(b.AddedDate) - new Date(a.AddedDate)	×
179	})
180
181	break	×
182	}
183	}
184
185	if (includeSensitive) {	×
186	return foundBreaches	×
187	}
188	return foundBreaches.filter(	×
189	breach => !breach.IsSensitive	×
190	)
191	}
192
193	function getBreachByName (allBreaches, breachName) {
194	breachName = breachName.toLowerCase()	×
195	if (RENAMED_BREACHES.includes(breachName)) {	×
196	breachName = RENAMED_BREACHES_MAP[breachName]	×
197	}
198	const foundBreach = allBreaches.find(breach => breach.Name.toLowerCase() === breachName)	×
199	return foundBreach	×
200	}
201
202	function filterBreaches (breaches) {
203	return breaches.filter(	×
204	breach => !breach.IsRetired &&	×
205	!breach.IsSpamList &&
206	!breach.IsFabricated &&
207	breach.IsVerified &&
208	breach.Domain !== ''
209	)
210	}
211
212	function getLatestBreach (breaches) {
213	let latestBreach = {}	×
214	let latestBreachDateTime = new Date(0)	×
215	for (const breach of breaches) {	×
216	if (breach.IsSensitive) {	×
217	continue	×
218	}
219	const breachAddedDate = new Date(breach.AddedDate)	×
220	if (breachAddedDate > latestBreachDateTime) {	×
221	latestBreachDateTime = breachAddedDate	×
222	latestBreach = breach	×
223	}
224	}
225	return latestBreach	×
226	}
227	/**
228	A range can be subscribed for callbacks with the following request:
229	POST /range/subscribe
230	{
231	hashPrefix:"[hash prefix]"
232	}
233	There are two possible response codes that will be returned:
234	1. HTTP 201: New range subscription has been created
235	2. HTTP 200: Range subscription already exists
236	* @param {string} sha1 first 6 chars of sha1 of the email being subscribed
237	* @returns 200 or 201 response codes
238	*/
239	async function subscribeHash (sha1) {
240	const sha1Prefix = sha1.slice(0, 6).toUpperCase()	×
241	const path = '/range/subscribe'	×
242	const options = {	×
243	method: 'POST',
244	json: { hashPrefix: sha1Prefix }
245	}
246
247	return await kAnonReq(path, options)	×
248	}
249
250	export {
251	req,
252	kAnonReq,
253	matchFluentID,
254	formatDataClassesArray,
255	loadBreachesIntoApp,
256	getBreachesForEmail,
257	getBreachByName,
258	getAllBreachesFromDb,
259	filterBreaches,
260	getLatestBreach,
261	subscribeHash
262	}

mozilla / blurts-server / #11668

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous