#4787

Committed 25 Oct 2023 07:28PM CUT coverage: 22.093% (-0.03%) from 22.121%

Build # #4787

Build Type

push

coveralls-python

Committed by

suhaibmujahid

Commit Message

Merge remote-tracking branch 'upstream/master' into crash-kind

Run Details

716 of 3560 branches covered (0.0%)

10 of 10 new or added lines in 1 file covered. (100.0%)

1925 of 8713 relevant lines covered (22.09%)

0.22 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/bugbot/rules/file_crash_bug.py

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.

import pprint
from functools import cached_property

import humanize
import jinja2
import requests

from bugbot import logger, utils
from bugbot.bug.analyzer import BugAnalyzer
from bugbot.bzcleaner import BzCleaner
from bugbot.crash import socorro_util
from bugbot.crash.analyzer import SignatureAnalyzer, SignaturesDataFetcher
from bugbot.user_activity import UserActivity, UserStatus


class FileCrashBug(BzCleaner):
    """File bugs for new actionable crashes."""

    MAX_BUG_TITLE_LENGTH = 255

    def __init__(self):
        super().__init__()

        self.bug_description_template = jinja2.Environment(
            loader=jinja2.FileSystemLoader("templates")
        ).get_template("file_crash_bug_description.md.jinja")

    def description(self):
        return "New actionable crashes"

    def columns(self):
        return ["component", "id", "summary"]

    def get_bz_params(self, date):
        return {
            "resolution": ["---", "FIXED"],
            "keywords": ["feature", "regression"],
            "keywords_type": "allwords",
        }

    @cached_property
    def nightly_version(self) -> int:
        """The version for the nightly channel as defined on Bugzilla."""
        return utils.get_nightly_version_from_bz()

    def _active_regression_authors(
        self, signatures: list[SignatureAnalyzer]
    ) -> set[str]:
        """Get Bugzilla usernames for users who are active and can be needinfo'd.

        Args:
            signatures: a list of signatures for which to check the status of
                their regression author.

        Returns:
            A set of user emails.
        """
        ni_skiplist = self.get_auto_ni_skiplist()
        users = UserActivity(include_fields=["requests"]).check_users(
            (
                signature.regressed_by_author["name"]
                for signature in signatures
                if signature.regressed_by_author
            ),
            keep_active=True,
            fetch_employee_info=True,
        )

        return {
            name
            for name, user in users.items()
            if name not in ni_skiplist
            and user["status"] == UserStatus.ACTIVE
            and not user["requests"]["needinfo"]["blocked"]
        }

    def get_bugs(self, date):
        self.query_url = None
        bugs = {}

        data_fetcher = SignaturesDataFetcher.find_new_actionable_crashes(
            "Firefox", "nightly"
        )
        signatures = data_fetcher.analyze()

        signature_details_delta = humanize.naturaldelta(data_fetcher.SUMMARY_DURATION)

        active_regression_authors = self._active_regression_authors(signatures)

        for signature in signatures:
            logger.debug("Generating bug for signature: %s", signature.signature_term)

            title = (
                f"Startup crash in [@ {signature.signature_term}]"
                if signature.is_startup_related_crash
                else f"Crash in [@ {signature.signature_term}]"
            )
            if len(title) > self.MAX_BUG_TITLE_LENGTH:
                title = title[: self.MAX_BUG_TITLE_LENGTH - 3] + "..."

            # Whether we should needinfo the regression author.
            needinfo_regression_author = (
                signature.regressed_by
                and signature.regressed_by_author["email"] in active_regression_authors
            )

            report = signature.fetch_representative_processed_crash()
            description = self.bug_description_template.render(
                {
                    **socorro_util.generate_bug_description_data(report),
                    "signature": signature,
                    "needinfo_regression_author": needinfo_regression_author,
                    "signature_details_delta": signature_details_delta,
                    "signature_details_channel": "Nightly",
                }
            )

            # TODO: Provide the following information:
            # [X] Crash signature
            # [X] Top 10 frames of crashing thread
            # [X] Component
            # [X] The kind of crash
            # [ ] Regression window
            # [X] Inducing patch
            # [X] Reason
            # [X] Regressed by
            # [X] Platform
            # [ ] Firefox status flags
            # [ ] Severity
            # [ ] Time correlation
            # [X] User comments
            # [X] Crash address commonalities
            # [ ] Estimated future crash volume

            bug_data = {
                "blocks": "bugbot-auto-crash",
                "type": "defect",
                "keywords": ["crash"],
                "summary": title,
                "product": signature.crash_component.product,
                "component": signature.crash_component.name,
                "op_sys": signature.bugzilla_op_sys,
                "rep_platform": signature.bugzilla_cpu_arch,
                "cf_crash_signature": f"[@ {signature.signature_term}]",
                "description": description,
                # NOTE(suhaib): the following CC is for testing purposes only
                # to allow us access and evaluate security bugs. It should be
                # removed at some point after we move to production.
                "cc": [
                    "smujahid@mozilla.com",
                    "mcastelluccio@mozilla.com",
                    "aryx.bugmail@gmx-topmail.de",
                ],
            }

            if needinfo_regression_author:
                bug_data["flags"] = [
                    {
                        "name": "needinfo",
                        "requestee": signature.regressed_by_author["name"],
                        "status": "?",
                        "new": "true",
                    }
                ]

            if signature.regressed_by:
                bug_data["keywords"].append("regression")

            if signature.regressed_by:
                bug_data["regressed_by"] = [signature.regressed_by]

            if signature.is_potential_security_crash:
                bug_data["groups"] = ["core-security"]

            if "regressed_by" in bug_data:
                bug_analyzer = BugAnalyzer(bug_data, signature.bugs_store)
                updates = bug_analyzer.detect_version_status_updates()
                for update in updates:
                    bug_data[update.flag] = update.status

                if bug_data["regressed_by"] and not updates:
                    # If we don't set the nightly flag here, the bot will set it
                    # later as part of `regression_new_set_nightly_affected` rule.
                    nightly_flag = utils.get_flag(
                        self.nightly_version, "status", "nightly"
                    )
                    bug_data[nightly_flag] = "affected"

            if self.dryrun:
                logger.info("Dry-run bug:")
                pprint.pprint(bug_data)
                bug_id = str(len(bugs) + 1)
            else:
                try:
                    bug = utils.create_bug(bug_data)
                except requests.HTTPError as err:
                    logger.exception(
                        "Failed to create a bug for signature %s: %s",
                        signature.signature_term,
                        err.response.text,
                    )
                    continue

                bug_id = str(bug["id"])
                # TODO: log the created bugs info somewhere (e.g., DB,
                # spreadsheet, or LabelStudio)

            bugs[bug_id] = {
                "id": bug_id,
                "summary": "..." if signature.is_potential_security_crash else title,
                "component": signature.crash_component,
            }

        logger.debug("Total of %d bugs have been filed", len(bugs))

        return bugs


if __name__ == "__main__":
    FileCrashBug().run()

1	# This Source Code Form is subject to the terms of the Mozilla Public
2	# License, v. 2.0. If a copy of the MPL was not distributed with this file,
3	# You can obtain one at http://mozilla.org/MPL/2.0/.
4
5	import pprint	×
6	from functools import cached_property	×
7
8	import humanize	×
9	import jinja2	×
10	import requests	×
11
12	from bugbot import logger, utils	×
13	from bugbot.bug.analyzer import BugAnalyzer	×
14	from bugbot.bzcleaner import BzCleaner	×
15	from bugbot.crash import socorro_util	×
16	from bugbot.crash.analyzer import SignatureAnalyzer, SignaturesDataFetcher	×
17	from bugbot.user_activity import UserActivity, UserStatus	×
18
19
20	class FileCrashBug(BzCleaner):	×
21	"""File bugs for new actionable crashes."""
22
23	MAX_BUG_TITLE_LENGTH = 255	×
24
25	def __init__(self):	×
26	super().__init__()	×
27
28	self.bug_description_template = jinja2.Environment(	×
29	loader=jinja2.FileSystemLoader("templates")
30	).get_template("file_crash_bug_description.md.jinja")
31
32	def description(self):	×
33	return "New actionable crashes"	×
34
35	def columns(self):	×
36	return ["component", "id", "summary"]	×
37
38	def get_bz_params(self, date):	×
39	return {	×
40	"resolution": ["---", "FIXED"],
41	"keywords": ["feature", "regression"],
42	"keywords_type": "allwords",
43	}
44
45	@cached_property	×
46	def nightly_version(self) -> int:	×
47	"""The version for the nightly channel as defined on Bugzilla."""
48	return utils.get_nightly_version_from_bz()	×
49
50	def _active_regression_authors(	×
51	self, signatures: list[SignatureAnalyzer]
52	) -> set[str]:
53	"""Get Bugzilla usernames for users who are active and can be needinfo'd.
54
55	Args:
56	signatures: a list of signatures for which to check the status of
57	their regression author.
58
59	Returns:
60	A set of user emails.
61	"""
62	ni_skiplist = self.get_auto_ni_skiplist()	×
63	users = UserActivity(include_fields=["requests"]).check_users(	×
64	(
65	signature.regressed_by_author["name"]
66	for signature in signatures
67	if signature.regressed_by_author
68	),
69	keep_active=True,
70	fetch_employee_info=True,
71	)
72
73	return {	×
74	name
75	for name, user in users.items()
76	if name not in ni_skiplist
77	and user["status"] == UserStatus.ACTIVE
78	and not user["requests"]["needinfo"]["blocked"]
79	}
80
81	def get_bugs(self, date):	×
82	self.query_url = None	×
83	bugs = {}	×
84
85	data_fetcher = SignaturesDataFetcher.find_new_actionable_crashes(	×
86	"Firefox", "nightly"
87	)
88	signatures = data_fetcher.analyze()	×
89
90	signature_details_delta = humanize.naturaldelta(data_fetcher.SUMMARY_DURATION)	×
91
92	active_regression_authors = self._active_regression_authors(signatures)	×
93
94	for signature in signatures:	×
95	logger.debug("Generating bug for signature: %s", signature.signature_term)	×
96
97	title = (	×
98	f"Startup crash in [@ {signature.signature_term}]"
99	if signature.is_startup_related_crash
100	else f"Crash in [@ {signature.signature_term}]"
101	)
102	if len(title) > self.MAX_BUG_TITLE_LENGTH:	×
103	title = title[: self.MAX_BUG_TITLE_LENGTH - 3] + "..."	×
104
105	# Whether we should needinfo the regression author.
106	needinfo_regression_author = (	×
107	signature.regressed_by
108	and signature.regressed_by_author["email"] in active_regression_authors
109	)
110
111	report = signature.fetch_representative_processed_crash()	×
112	description = self.bug_description_template.render(	×
113	{
114	**socorro_util.generate_bug_description_data(report),
115	"signature": signature,
116	"needinfo_regression_author": needinfo_regression_author,
117	"signature_details_delta": signature_details_delta,
118	"signature_details_channel": "Nightly",
119	}
120	)
121
122	# TODO: Provide the following information:
123	# [X] Crash signature
124	# [X] Top 10 frames of crashing thread
125	# [X] Component
126	# [X] The kind of crash
127	# [ ] Regression window
128	# [X] Inducing patch
129	# [X] Reason
130	# [X] Regressed by
131	# [X] Platform
132	# [ ] Firefox status flags
133	# [ ] Severity
134	# [ ] Time correlation
135	# [X] User comments
136	# [X] Crash address commonalities
137	# [ ] Estimated future crash volume
138
139	bug_data = {	×
140	"blocks": "bugbot-auto-crash",
141	"type": "defect",
142	"keywords": ["crash"],
143	"summary": title,
144	"product": signature.crash_component.product,
145	"component": signature.crash_component.name,
146	"op_sys": signature.bugzilla_op_sys,
147	"rep_platform": signature.bugzilla_cpu_arch,
148	"cf_crash_signature": f"[@ {signature.signature_term}]",
149	"description": description,
150	# NOTE(suhaib): the following CC is for testing purposes only
151	# to allow us access and evaluate security bugs. It should be
152	# removed at some point after we move to production.
153	"cc": [
154	"smujahid@mozilla.com",
155	"mcastelluccio@mozilla.com",
156	"aryx.bugmail@gmx-topmail.de",
157	],
158	}
159
160	if needinfo_regression_author:	×
161	bug_data["flags"] = [	×
162	{
163	"name": "needinfo",
164	"requestee": signature.regressed_by_author["name"],
165	"status": "?",
166	"new": "true",
167	}
168	]
169
170	if signature.regressed_by:	×
171	bug_data["keywords"].append("regression")	×
172
173	if signature.regressed_by:	×
174	bug_data["regressed_by"] = [signature.regressed_by]	×
175
176	if signature.is_potential_security_crash:	×
177	bug_data["groups"] = ["core-security"]	×
178
179	if "regressed_by" in bug_data:	×
180	bug_analyzer = BugAnalyzer(bug_data, signature.bugs_store)	×
181	updates = bug_analyzer.detect_version_status_updates()	×
182	for update in updates:	×
183	bug_data[update.flag] = update.status	×
184
185	if bug_data["regressed_by"] and not updates:	×
186	# If we don't set the nightly flag here, the bot will set it
187	# later as part of `regression_new_set_nightly_affected` rule.
188	nightly_flag = utils.get_flag(	×
189	self.nightly_version, "status", "nightly"
190	)
191	bug_data[nightly_flag] = "affected"	×
192
193	if self.dryrun:	×
194	logger.info("Dry-run bug:")	×
195	pprint.pprint(bug_data)	×
196	bug_id = str(len(bugs) + 1)	×
197	else:
198	try:	×
199	bug = utils.create_bug(bug_data)	×
200	except requests.HTTPError as err:	×
201	logger.exception(	×
202	"Failed to create a bug for signature %s: %s",
203	signature.signature_term,
204	err.response.text,
205	)
206	continue	×
207
208	bug_id = str(bug["id"])	×
209	# TODO: log the created bugs info somewhere (e.g., DB,
210	# spreadsheet, or LabelStudio)
211
212	bugs[bug_id] = {	×
213	"id": bug_id,
214	"summary": "..." if signature.is_potential_security_crash else title,
215	"component": signature.crash_component,
216	}
217
218	logger.debug("Total of %d bugs have been filed", len(bugs))	×
219
220	return bugs	×
221
222
223	if __name__ == "__main__":	×
224	FileCrashBug().run()	×

mozilla / relman-auto-nag / #4787

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous