5487713236

pending completion

Build # 5487713236

Build Type

Pull #597

github

Committed by

web-flow

Commit Message

Merge f2aedefe6 into 14755187f

Pull Request Pull Request #597: Second order sql injection fixes

Run Details

49 of 336 branches covered (14.58%)

Branch coverage included in aggregate %.

5 of 10 new or added lines in 2 files covered. (50.0%)

1 existing line in 1 file now uncovered.

400 of 1361 relevant lines covered (29.39%)

1.47 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

9.68

/src/pe_reports/pages.py

"""Collect and distribute graphical data to readable charts in the presentation."""

# Standard Python Libraries
import datetime
import logging
import os
import re

# Third-Party Libraries
import chevron

from .charts import Charts

# Import Classes
from .metrics import Credentials, Cyber_Six, Domains_Masqs, Malware_Vulns


# Style and build tables
def buildTable(df, classList, sizingList=[]):
    """Build HTML tables from a pandas dataframe."""
    # SizingList specifies the proportional width of each column.
    # The number of integers in the list must equal the number of
    # columns in the dataframe AND add up to 100
    if not sizingList:
        average = 100 / len(df.columns)
        sizingList = [average] * len(df.columns)
    headers = """<table border="1" class="{classes}">\n<thead>\n""".format(
        classes=", ".join(classList)
    )
    headers += '<tr style="text-align: right;">'
    for head in df.columns:
        headers += "<th>" + head + "</th>\n"
    headers += "</tr>\n</thead>"
    html = ""
    body = "<tbody>\n"
    counter = 0
    for row in df.itertuples(index=False):
        if counter % 2 == 0:
            body += '<tr class="even">\n'
        else:
            body += '<tr class="odd">\n'
        for col in range(0, len(df.columns)):
            body += (
                "<td style='width:{size}%'>".format(size=str(sizingList[col]))
                + str(row[col])
                + "</td>\n"
            )

        body += "</tr>\n"
        counter += 1
    body += "</tbody>\n</table>"
    html = headers + body
    return html


def buildAppendixList(df):
    """Build report appendix."""
    html = "<div> \n"

    for row in df.itertuples(index=False):
        html += """<p class="content"><b style="font-size: 15px;">{breach_name}</b><br>{description}
        </p>\n""".format(
            breach_name=row[0], description=row[1]
        )
    html += "\n</div>"
    return html


def sanitize_uid(string):
    """Remove special characters from uids."""
    return re.sub(r"[^a-zA-Z0-9\-]", "", string)


def credential(chevron_dict, trending_start_date, start_date, end_date, org_uid):
    """Build exposed credential page."""
    Credential = Credentials(
        trending_start_date, start_date, end_date, sanitize_uid(org_uid)
    )
    # Build exposed credential stacked bar chart
    width = 24
    height = 9.5
    name = "inc_date_df"
    title = "Trending Exposures by Week"
    x_label = "Week Reported"
    y_label = "Creds Exposed"
    cred_date_chart = Charts(
        Credential.by_week(),
        width,
        height,
        name,
        title,
        x_label,
        y_label,
    )
    cred_date_chart.line_chart()
    breach_table = buildTable(Credential.breach_details(), ["table"])

    creds_dict = {
        "breach": Credential.breaches(),
        "creds": Credential.total(),
        "pw_creds": Credential.password(),
        "breach_table": breach_table,
        "breachAppendix": buildAppendixList(Credential.breach_appendix()),
    }
    chevron_dict.update(creds_dict)

    return chevron_dict, Credential.creds_view


def masquerading(chevron_dict, start_date, end_date, org_uid):
    """Build masquerading page."""
    Domain_Masq = Domains_Masqs(start_date, end_date, sanitize_uid(org_uid))
    chevron_dict.update(
        {
            "domain_table": buildTable(Domain_Masq.summary(), ["table"], []),
            "suspectedDomains": Domain_Masq.count(),
            "uniqueTlds": Domain_Masq.utlds(),
        }
    )
    return chevron_dict, Domain_Masq.df_mal


def mal_vuln(chevron_dict, start_date, end_date, org_uid):
    """Build Malwares and Vulnerabilities page."""
    Malware_Vuln = Malware_Vulns(start_date, end_date, org_uid)
    # Build insecure protocol horizontal bar chart
    width = 9
    height = 4.7
    name = "pro_count"
    title = ""
    x_label = ""
    y_label = ""
    protocol_chart = Charts(
        Malware_Vuln.protocol_count(),
        width,
        height,
        name,
        title,
        x_label,
        y_label,
    )
    protocol_chart.h_bar()
    # Build unverified vulnerability horizontal bar chart
    width = 9
    height = 4.7
    name = "unverif_vuln_count"
    title = ""
    x_label = "Unverified CVEs"
    y_label = ""
    unverif_vuln_chart = Charts(
        Malware_Vuln.unverified_cve(),
        width,
        height,
        name,
        title,
        x_label,
        y_label,
    )
    unverif_vuln_chart.h_bar()
    # Build tables
    risky_assets = Malware_Vuln.isolate_risky_assets(Malware_Vuln.insecure_df)
    risky_assets = risky_assets[:7]
    risky_assets.columns = ["IP", "Protocol"]
    risky_assets_table = buildTable(risky_assets, ["table"], [50, 50])
    verif_vulns = Malware_Vuln.verif_vulns()
    verif_vulns.columns = ["CVE", "IP", "Port"]
    verif_vulns_table = buildTable(verif_vulns, ["table"], [40, 40, 20])
    verif_vulns_summary_table = buildTable(
        Malware_Vuln.verif_vulns_summary(), ["table"], [15, 15, 15, 55]
    )

    # Update chevron dictionary
    vulns_dict = {
        "verif_vulns": verif_vulns_table,
        "verif_vulns_summary": verif_vulns_summary_table,
        "risky_assets": risky_assets_table,
        "riskyPorts": Malware_Vuln.risky_ports_count(),
        "verifVulns": Malware_Vuln.total_verif_vulns(),
        "unverifVulns": Malware_Vuln.unverified_vuln_count(),
    }
    chevron_dict.update(vulns_dict)
    return (
        chevron_dict,
        Malware_Vuln.insecure_df,
        Malware_Vuln.vulns_df,
        Malware_Vuln.assets_df,
    )


def dark_web(chevron_dict, trending_start_date, start_date, end_date, org_uid):
    """Dark Web Mentions."""
    Cyber6 = Cyber_Six(trending_start_date, start_date, end_date, org_uid)
    # Build dark web mentions over time line chart
    width = 19
    height = 9
    name = "web_only_df_2"
    title = ""
    x_label = "Dark Web Mentions"
    y_label = "Mentions count"
    dark_mentions_chart = Charts(
        Cyber6.dark_web_date(),
        width,
        height,
        name,
        title,
        x_label,
        y_label,
    )
    dark_mentions_chart.line_chart()
    # Build forum type / conversation content pie chart
    width = 19
    height = 9
    name = "dark_web_forum_pie"
    title = ""
    x_label = ""
    y_label = ""
    pie_chart = Charts(
        Cyber6.dark_web_content(),
        width,
        height,
        name,
        title,
        x_label,
        y_label,
    )
    pie_chart.pie()

    # Limit the number of rows for large dataframes
    dark_web_bad_actors = Cyber6.dark_web_bad_actors()[:10]
    alert_exec = Cyber6.alerts_exec()[:8]

    # Build tables
    dark_web_sites_table = buildTable(Cyber6.dark_web_sites(), ["table"], [50, 50])
    alerts_threats_table = buildTable(Cyber6.alerts_threats(), ["table"], [40, 40, 20])
    dark_web_actors_table = buildTable(dark_web_bad_actors, ["table"], [50, 50])
    dark_web_tags_table = buildTable(Cyber6.dark_web_tags(), ["table"], [60, 40])
    alerts_exec_table = buildTable(alert_exec, ["table"], [15, 70, 15])
    dark_web_act_table = buildTable(Cyber6.dark_web_most_act(), ["table"], [75, 25])
    alerts_site_table = buildTable(Cyber6.alerts_site(), ["table"], [50, 50])
    top_cves_table = buildTable(Cyber6.top_cve_table(), ["table"], [30, 70])

    dark_web_dict = {
        "darkWeb": Cyber6.dark_web_count(),
        "dark_web_sites": dark_web_sites_table,
        "alerts_threats": alerts_threats_table,
        "dark_web_actors": dark_web_actors_table,
        "dark_web_tags": dark_web_tags_table,
        "alerts_exec": alerts_exec_table,
        "dark_web_act": dark_web_act_table,
        "alerts_site": alerts_site_table,
        "top_cves": top_cves_table,
    }

    chevron_dict.update(dark_web_dict)
    return (chevron_dict, Cyber6.dark_web_mentions, Cyber6.alerts, Cyber6.top_cves)


def init(datestring, org_name, org_uid):
    """Call each page of the report."""
    # Format start_date and end_date for the bi-monthly reporting period.
    # If the given end_date is the 15th, then the start_date is the 1st.
    # Otherwise, the start_date will be the 16th of the respective month.

    # Load source HTML
    try:
        basedir = os.path.abspath(os.path.dirname(__file__))
        template = os.path.join(basedir, "template.html")
        file = open(template)
        source_html = file.read().replace("\n", " ")
        # Close PDF
        file.close()
    except FileNotFoundError:
        logging.error("Template cannot be found. It must be named: '%s'", template)
        return 1

    end_date = datetime.datetime.strptime(datestring, "%Y-%m-%d").date()
    if end_date.day == 15:
        start_date = datetime.datetime(end_date.year, end_date.month, 1)
    else:
        start_date = datetime.datetime(end_date.year, end_date.month, 16)
    # create the trending start date which is 4 weeks from the last day of the report period
    # 27 days plus the last day is 4 weeks
    days = datetime.timedelta(27)
    trending_start_date = end_date - days
    start = start_date.strftime("%m/%d/%Y")
    end = end_date.strftime("%m/%d/%Y")
    chevron_dict = {
        "department": org_name,
        "dateRange": start + " - " + end,
        "endDate": end,
    }

    chevron_dict, creds_sum = credential(
        chevron_dict, trending_start_date, start_date, end_date, org_uid
    )

    chevron_dict, masq_df = masquerading(chevron_dict, start_date, end_date, org_uid)

    chevron_dict, insecure_df, vulns_df, assets_df = mal_vuln(
        chevron_dict, start_date, end_date, org_uid
    )

    chevron_dict, dark_web_mentions, alerts, top_cves = dark_web(
        chevron_dict, trending_start_date, start_date, end_date, org_uid
    )

    html = chevron.render(source_html, chevron_dict)

    return (
        html,
        creds_sum,
        masq_df,
        insecure_df,
        vulns_df,
        assets_df,
        dark_web_mentions,
        alerts,
        top_cves,
    )

1	"""Collect and distribute graphical data to readable charts in the presentation."""
2
3	# Standard Python Libraries
4	import datetime	5✔
5	import logging	5✔
6	import os	5✔
7	import re	5✔
8
9	# Third-Party Libraries
10	import chevron	5✔
11
12	from .charts import Charts	5✔
13
14	# Import Classes
15	from .metrics import Credentials, Cyber_Six, Domains_Masqs, Malware_Vulns	5✔
16
17
18	# Style and build tables
19	def buildTable(df, classList, sizingList=[]):	5✔
20	"""Build HTML tables from a pandas dataframe."""
21	# SizingList specifies the proportional width of each column.
22	# The number of integers in the list must equal the number of
23	# columns in the dataframe AND add up to 100
24	if not sizingList:	×
25	average = 100 / len(df.columns)	×
26	sizingList = [average] * len(df.columns)	×
27	headers = """<table border="1" class="{classes}">\n<thead>\n""".format(	×
28	classes=", ".join(classList)
29	)
30	headers += '<tr style="text-align: right;">'	×
31	for head in df.columns:	×
32	headers += "<th>" + head + "</th>\n"	×
33	headers += "</tr>\n</thead>"	×
34	html = ""	×
35	body = "<tbody>\n"	×
36	counter = 0	×
37	for row in df.itertuples(index=False):	×
38	if counter % 2 == 0:	×
39	body += '<tr class="even">\n'	×
40	else:
41	body += '<tr class="odd">\n'	×
42	for col in range(0, len(df.columns)):	×
43	body += (	×
44	"<td style='width:{size}%'>".format(size=str(sizingList[col]))
45	+ str(row[col])
46	+ "</td>\n"
47	)
48
49	body += "</tr>\n"	×
50	counter += 1	×
51	body += "</tbody>\n</table>"	×
52	html = headers + body	×
53	return html	×
54
55
56	def buildAppendixList(df):	5✔
57	"""Build report appendix."""
58	html = "<div> \n"	×
59
60	for row in df.itertuples(index=False):	×
61	html += """<p class="content"><b style="font-size: 15px;">{breach_name}</b><br>{description}	×
62	</p>\n""".format(
63	breach_name=row[0], description=row[1]
64	)
65	html += "\n</div>"	×
66	return html	×
67
68
69	def sanitize_uid(string):	5✔
70	"""Remove special characters from uids."""
NEW 71	return re.sub(r"[^a-zA-Z0-9\-]", "", string)	×
72
73
74	def credential(chevron_dict, trending_start_date, start_date, end_date, org_uid):	5✔
75	"""Build exposed credential page."""
NEW 76	Credential = Credentials(	×
77	trending_start_date, start_date, end_date, sanitize_uid(org_uid)
78	)
79	# Build exposed credential stacked bar chart
80	width = 24	×
81	height = 9.5	×
82	name = "inc_date_df"	×
83	title = "Trending Exposures by Week"	×
84	x_label = "Week Reported"	×
85	y_label = "Creds Exposed"	×
86	cred_date_chart = Charts(	×
87	Credential.by_week(),
88	width,
89	height,
90	name,
91	title,
92	x_label,
93	y_label,
94	)
95	cred_date_chart.line_chart()	×
96	breach_table = buildTable(Credential.breach_details(), ["table"])	×
97
98	creds_dict = {	×
99	"breach": Credential.breaches(),
100	"creds": Credential.total(),
101	"pw_creds": Credential.password(),
102	"breach_table": breach_table,
103	"breachAppendix": buildAppendixList(Credential.breach_appendix()),
104	}
105	chevron_dict.update(creds_dict)	×
106
107	return chevron_dict, Credential.creds_view	×
108
109
110	def masquerading(chevron_dict, start_date, end_date, org_uid):	5✔
111	"""Build masquerading page."""
NEW 112	Domain_Masq = Domains_Masqs(start_date, end_date, sanitize_uid(org_uid))	×
113	chevron_dict.update(	×
114	{
115	"domain_table": buildTable(Domain_Masq.summary(), ["table"], []),
116	"suspectedDomains": Domain_Masq.count(),
117	"uniqueTlds": Domain_Masq.utlds(),
118	}
119	)
120	return chevron_dict, Domain_Masq.df_mal	×
121
122
123	def mal_vuln(chevron_dict, start_date, end_date, org_uid):	5✔
124	"""Build Malwares and Vulnerabilities page."""
125	Malware_Vuln = Malware_Vulns(start_date, end_date, org_uid)	×
126	# Build insecure protocol horizontal bar chart
127	width = 9	×
128	height = 4.7	×
129	name = "pro_count"	×
130	title = ""	×
131	x_label = ""	×
132	y_label = ""	×
133	protocol_chart = Charts(	×
134	Malware_Vuln.protocol_count(),
135	width,
136	height,
137	name,
138	title,
139	x_label,
140	y_label,
141	)
142	protocol_chart.h_bar()	×
143	# Build unverified vulnerability horizontal bar chart
144	width = 9	×
145	height = 4.7	×
146	name = "unverif_vuln_count"	×
147	title = ""	×
148	x_label = "Unverified CVEs"	×
149	y_label = ""	×
150	unverif_vuln_chart = Charts(	×
151	Malware_Vuln.unverified_cve(),
152	width,
153	height,
154	name,
155	title,
156	x_label,
157	y_label,
158	)
159	unverif_vuln_chart.h_bar()	×
160	# Build tables
161	risky_assets = Malware_Vuln.isolate_risky_assets(Malware_Vuln.insecure_df)	×
162	risky_assets = risky_assets[:7]	×
163	risky_assets.columns = ["IP", "Protocol"]	×
164	risky_assets_table = buildTable(risky_assets, ["table"], [50, 50])	×
165	verif_vulns = Malware_Vuln.verif_vulns()	×
166	verif_vulns.columns = ["CVE", "IP", "Port"]	×
167	verif_vulns_table = buildTable(verif_vulns, ["table"], [40, 40, 20])	×
168	verif_vulns_summary_table = buildTable(	×
169	Malware_Vuln.verif_vulns_summary(), ["table"], [15, 15, 15, 55]
170	)
171
172	# Update chevron dictionary
173	vulns_dict = {	×
174	"verif_vulns": verif_vulns_table,
175	"verif_vulns_summary": verif_vulns_summary_table,
176	"risky_assets": risky_assets_table,
177	"riskyPorts": Malware_Vuln.risky_ports_count(),
178	"verifVulns": Malware_Vuln.total_verif_vulns(),
179	"unverifVulns": Malware_Vuln.unverified_vuln_count(),
180	}
181	chevron_dict.update(vulns_dict)	×
182	return (	×
183	chevron_dict,
184	Malware_Vuln.insecure_df,
185	Malware_Vuln.vulns_df,
186	Malware_Vuln.assets_df,
187	)
188
189
190	def dark_web(chevron_dict, trending_start_date, start_date, end_date, org_uid):	5✔
191	"""Dark Web Mentions."""
192	Cyber6 = Cyber_Six(trending_start_date, start_date, end_date, org_uid)	×
193	# Build dark web mentions over time line chart
194	width = 19	×
195	height = 9	×
196	name = "web_only_df_2"	×
197	title = ""	×
198	x_label = "Dark Web Mentions"	×
199	y_label = "Mentions count"	×
200	dark_mentions_chart = Charts(	×
201	Cyber6.dark_web_date(),
202	width,
203	height,
204	name,
205	title,
206	x_label,
207	y_label,
208	)
209	dark_mentions_chart.line_chart()	×
210	# Build forum type / conversation content pie chart
211	width = 19	×
212	height = 9	×
213	name = "dark_web_forum_pie"	×
214	title = ""	×
215	x_label = ""	×
216	y_label = ""	×
217	pie_chart = Charts(	×
218	Cyber6.dark_web_content(),
219	width,
220	height,
221	name,
222	title,
223	x_label,
224	y_label,
225	)
226	pie_chart.pie()	×
227
228	# Limit the number of rows for large dataframes
229	dark_web_bad_actors = Cyber6.dark_web_bad_actors()[:10]	×
230	alert_exec = Cyber6.alerts_exec()[:8]	×
231
232	# Build tables
233	dark_web_sites_table = buildTable(Cyber6.dark_web_sites(), ["table"], [50, 50])	×
234	alerts_threats_table = buildTable(Cyber6.alerts_threats(), ["table"], [40, 40, 20])	×
235	dark_web_actors_table = buildTable(dark_web_bad_actors, ["table"], [50, 50])	×
236	dark_web_tags_table = buildTable(Cyber6.dark_web_tags(), ["table"], [60, 40])	×
237	alerts_exec_table = buildTable(alert_exec, ["table"], [15, 70, 15])	×
238	dark_web_act_table = buildTable(Cyber6.dark_web_most_act(), ["table"], [75, 25])	×
239	alerts_site_table = buildTable(Cyber6.alerts_site(), ["table"], [50, 50])	×
240	top_cves_table = buildTable(Cyber6.top_cve_table(), ["table"], [30, 70])	×
241
242	dark_web_dict = {	×
243	"darkWeb": Cyber6.dark_web_count(),
244	"dark_web_sites": dark_web_sites_table,
245	"alerts_threats": alerts_threats_table,
246	"dark_web_actors": dark_web_actors_table,
247	"dark_web_tags": dark_web_tags_table,
248	"alerts_exec": alerts_exec_table,
249	"dark_web_act": dark_web_act_table,
250	"alerts_site": alerts_site_table,
251	"top_cves": top_cves_table,
252	}
253
254	chevron_dict.update(dark_web_dict)	×
255	return (chevron_dict, Cyber6.dark_web_mentions, Cyber6.alerts, Cyber6.top_cves)	×
256
257
258	def init(datestring, org_name, org_uid):	5✔
259	"""Call each page of the report."""
260	# Format start_date and end_date for the bi-monthly reporting period.
261	# If the given end_date is the 15th, then the start_date is the 1st.
262	# Otherwise, the start_date will be the 16th of the respective month.
263
264	# Load source HTML
265	try:	×
266	basedir = os.path.abspath(os.path.dirname(__file__))	×
267	template = os.path.join(basedir, "template.html")	×
268	file = open(template)	×
269	source_html = file.read().replace("\n", " ")	×
270	# Close PDF
271	file.close()	×
272	except FileNotFoundError:	×
273	logging.error("Template cannot be found. It must be named: '%s'", template)	×
274	return 1	×
275
276	end_date = datetime.datetime.strptime(datestring, "%Y-%m-%d").date()	×
277	if end_date.day == 15:	×
278	start_date = datetime.datetime(end_date.year, end_date.month, 1)	×
279	else:
280	start_date = datetime.datetime(end_date.year, end_date.month, 16)	×
281	# create the trending start date which is 4 weeks from the last day of the report period
282	# 27 days plus the last day is 4 weeks
283	days = datetime.timedelta(27)	×
284	trending_start_date = end_date - days	×
285	start = start_date.strftime("%m/%d/%Y")	×
286	end = end_date.strftime("%m/%d/%Y")	×
287	chevron_dict = {	×
288	"department": org_name,
289	"dateRange": start + " - " + end,
290	"endDate": end,
291	}
292
293	chevron_dict, creds_sum = credential(	×
294	chevron_dict, trending_start_date, start_date, end_date, org_uid
295	)
296
297	chevron_dict, masq_df = masquerading(chevron_dict, start_date, end_date, org_uid)	×
298
299	chevron_dict, insecure_df, vulns_df, assets_df = mal_vuln(	×
300	chevron_dict, start_date, end_date, org_uid
301	)
302
303	chevron_dict, dark_web_mentions, alerts, top_cves = dark_web(	×
304	chevron_dict, trending_start_date, start_date, end_date, org_uid
305	)
306
307	html = chevron.render(source_html, chevron_dict)	×
308
309	return (	×
310	html,
311	creds_sum,
312	masq_df,
313	insecure_df,
314	vulns_df,
315	assets_df,
316	dark_web_mentions,
317	alerts,
318	top_cves,
319	)

cisagov / pe-reports / 5487713236

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous