#4841

Committed 05 Dec 2023 04:04PM UTC coverage: 21.936% (+0.05%) from 21.889%

Build # #4841

Build Type

push

coveralls-python

Committed by

web-flow

Commit Message

Bump recurring-ical-events from 2.0.2 to 2.1.1 (#2280)

Bumps [recurring-ical-events](https://github.com/niccokunzmann/python-recurring-ical-events) from 2.0.2 to 2.1.1.
- [Commits](https://github.com/niccokunzmann/python-recurring-ical-events/compare/v2.0.2...v2.1.1)

---
updated-dependencies:
- dependency-name: recurring-ical-events
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

Run Details

716 of 3588 branches covered (0.0%)

1924 of 8771 relevant lines covered (21.94%)

0.22 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/bugbot/rules/file_crash_bug.py

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.

import pprint
from functools import cached_property

import humanize
import jinja2
import requests
from libmozdata.bugzilla import Bugzilla

from bugbot import logger, utils
from bugbot.bug.analyzer import BugAnalyzer
from bugbot.bzcleaner import BzCleaner
from bugbot.crash import socorro_util
from bugbot.crash.analyzer import SignatureAnalyzer, SignaturesDataFetcher
from bugbot.user_activity import UserActivity, UserStatus


class FileCrashBug(BzCleaner):
    """File bugs for new actionable crashes."""

    MAX_BUG_TITLE_LENGTH = 255

    def __init__(self):
        super().__init__()

        self.bug_description_template = jinja2.Environment(
            loader=jinja2.FileSystemLoader("templates")
        ).get_template("file_crash_bug_description.md.jinja")

    def description(self):
        return "New actionable crashes"

    def columns(self):
        return ["component", "id", "summary"]

    def get_bz_params(self, date):
        return {
            "resolution": ["---", "FIXED"],
            "keywords": ["feature", "regression"],
            "keywords_type": "allwords",
        }

    @cached_property
    def current_status_flags(self) -> list[str]:
        """The status flags for Firefox, which includes Nightly, Beta, Release, and ESR."""
        status = "cf_status_firefox"
        status_esr = "cf_status_firefox_esr"
        data: dict[str, list[str]] = {
            status: [],
            status_esr: [],
        }

        def handler(bug):
            for field in bug.keys():
                if field.startswith(status_esr):
                    data[status_esr].append(field)
                elif field.startswith(status):
                    data[status].append(field)

        Bugzilla(bugids=1234567, include_fields="_custom", bughandler=handler).wait()

        return (
            # ESR
            sorted(
                data[status_esr],
                key=lambda flag: int(flag[len(status_esr) :]),
            )[-1:]
            # Release, Beta, Nightly
            + sorted(
                data[status],
                key=lambda flag: int(flag[len(status) :]),
            )[-3:]
        )

    @cached_property
    def nightly_status_flag(self) -> str:
        """The nightly release status flag for Firefox."""
        return self.current_status_flags[-1]

    def _active_regression_authors(
        self, signatures: list[SignatureAnalyzer]
    ) -> set[str]:
        """Get Bugzilla usernames for users who are active and can be needinfo'd.

        Args:
            signatures: a list of signatures for which to check the status of
                their regression author.

        Returns:
            A set of user emails.
        """
        ni_skiplist = self.get_auto_ni_skiplist()
        users = UserActivity(include_fields=["requests"]).check_users(
            (
                signature.regressed_by_author["name"]
                for signature in signatures
                if signature.regressed_by_author
            ),
            keep_active=True,
            fetch_employee_info=True,
        )

        return {
            name
            for name, user in users.items()
            if name not in ni_skiplist
            and user["status"] == UserStatus.ACTIVE
            and not user["requests"]["needinfo"]["blocked"]
        }

    def get_bugs(self, date):
        self.query_url = None
        bugs = {}

        data_fetcher = SignaturesDataFetcher.find_new_actionable_crashes(
            "Firefox", "nightly"
        )
        signatures = data_fetcher.analyze()
        # This is the last filtering stage which aims to avoid filing bugs for
        # junky crashes, where the volume is low and the crashes do not show
        # signals of being actionable or critical.
        signatures = [
            signature
            for signature in signatures
            if signature.num_installs > 5
            or signature.num_crashes > 25
            or signature.is_potential_near_null_crash
            or signature.is_potential_security_crash
            or signature.regressed_by
            or signature.has_moz_crash_reason("MOZ_RELEASE_ASSERT")
            or signature.has_moz_crash_reason("DocumentChannel::SetLoadFlags")
        ]

        signature_details_delta = humanize.naturaldelta(data_fetcher.SUMMARY_DURATION)

        active_regression_authors = self._active_regression_authors(signatures)

        for signature in signatures:
            logger.debug("Generating bug for signature: %s", signature.signature_term)

            title = (
                f"Startup crash in [@ {signature.signature_term}]"
                if signature.is_startup_related_crash
                else f"Crash in [@ {signature.signature_term}]"
            )
            if len(title) > self.MAX_BUG_TITLE_LENGTH:
                title = title[: self.MAX_BUG_TITLE_LENGTH - 3] + "..."

            # Whether we should needinfo the regression author.
            needinfo_regression_author = (
                signature.regressed_by
                and signature.regressed_by_author["email"] in active_regression_authors
            )

            report = signature.fetch_representative_processed_crash()
            description = self.bug_description_template.render(
                {
                    **socorro_util.generate_bug_description_data(report),
                    "signature": signature,
                    "needinfo_regression_author": needinfo_regression_author,
                    "signature_details_delta": signature_details_delta,
                    "signature_details_channel": "Nightly",
                }
            )

            # TODO: Provide the following information:
            # [X] Crash signature
            # [X] Top 10 frames of crashing thread
            # [X] Component
            # [X] The kind of crash
            # [ ] Regression window
            # [X] Inducing patch
            # [X] Reason
            # [X] Regressed by
            # [X] Platform
            # [x] Firefox status flags
            # [ ] Severity
            # [ ] Time correlation
            # [X] User comments
            # [X] Crash address commonalities
            # [ ] Estimated future crash volume

            bug_data = {
                "blocks": "bugbot-auto-crash",
                "type": "defect",
                "keywords": ["crash"],
                "summary": title,
                "product": signature.crash_component.product,
                "component": signature.crash_component.name,
                "op_sys": signature.bugzilla_op_sys,
                "rep_platform": signature.bugzilla_cpu_arch,
                "cf_crash_signature": f"[@ {signature.signature_term}]",
                "description": description,
                self.nightly_status_flag: "affected",
                # NOTE(suhaib): the following CC is for testing purposes only
                # to allow us access and evaluate security bugs. It should be
                # removed at some point after we move to production.
                "cc": [
                    "smujahid@mozilla.com",
                    "mcastelluccio@mozilla.com",
                    "aryx.bugmail@gmx-topmail.de",
                ],
            }

            if needinfo_regression_author:
                bug_data["flags"] = [
                    {
                        "name": "needinfo",
                        "requestee": signature.regressed_by_author["name"],
                        "status": "?",
                        "new": "true",
                    }
                ]
                bug_data["cc"].append(signature.regressed_by_author["name"])

            if signature.is_potential_security_crash:
                bug_data["groups"] = ["core-security"]

            if signature.regressed_by:
                bug_data["keywords"].append("regression")
                bug_data["regressed_by"] = [signature.regressed_by]

                # Empty statuses are needed to detect the affected releases.
                for flag in self.current_status_flags:
                    if flag not in bug_data:
                        bug_data[flag] = "---"

                bug_analyzer = BugAnalyzer(bug_data, signature.bugs_store)
                updates = bug_analyzer.detect_version_status_updates()
                for update in updates:
                    bug_data[update.flag] = update.status

            if self.dryrun:
                logger.info("Dry-run bug:")
                pprint.pprint(bug_data)
                bug_id = str(len(bugs) + 1)
            else:
                try:
                    bug = utils.create_bug(bug_data)
                except requests.HTTPError as err:
                    logger.exception(
                        "Failed to create a bug for signature %s: %s",
                        signature.signature_term,
                        err.response.text,
                    )
                    continue

                bug_id = str(bug["id"])
                # TODO: log the created bugs info somewhere (e.g., DB,
                # spreadsheet, or LabelStudio)

            bugs[bug_id] = {
                "id": bug_id,
                "summary": "..." if signature.is_potential_security_crash else title,
                "component": signature.crash_component,
            }

        logger.debug("Total of %d bugs have been filed", len(bugs))

        return bugs


if __name__ == "__main__":
    FileCrashBug().run()

1	# This Source Code Form is subject to the terms of the Mozilla Public
2	# License, v. 2.0. If a copy of the MPL was not distributed with this file,
3	# You can obtain one at http://mozilla.org/MPL/2.0/.
4
5	import pprint	×
6	from functools import cached_property	×
7
8	import humanize	×
9	import jinja2	×
10	import requests	×
11	from libmozdata.bugzilla import Bugzilla	×
12
13	from bugbot import logger, utils	×
14	from bugbot.bug.analyzer import BugAnalyzer	×
15	from bugbot.bzcleaner import BzCleaner	×
16	from bugbot.crash import socorro_util	×
17	from bugbot.crash.analyzer import SignatureAnalyzer, SignaturesDataFetcher	×
18	from bugbot.user_activity import UserActivity, UserStatus	×
19
20
21	class FileCrashBug(BzCleaner):	×
22	"""File bugs for new actionable crashes."""
23
24	MAX_BUG_TITLE_LENGTH = 255	×
25
26	def __init__(self):	×
27	super().__init__()	×
28
29	self.bug_description_template = jinja2.Environment(	×
30	loader=jinja2.FileSystemLoader("templates")
31	).get_template("file_crash_bug_description.md.jinja")
32
33	def description(self):	×
34	return "New actionable crashes"	×
35
36	def columns(self):	×
37	return ["component", "id", "summary"]	×
38
39	def get_bz_params(self, date):	×
40	return {	×
41	"resolution": ["---", "FIXED"],
42	"keywords": ["feature", "regression"],
43	"keywords_type": "allwords",
44	}
45
46	@cached_property	×
47	def current_status_flags(self) -> list[str]:	×
48	"""The status flags for Firefox, which includes Nightly, Beta, Release, and ESR."""
49	status = "cf_status_firefox"	×
50	status_esr = "cf_status_firefox_esr"	×
51	data: dict[str, list[str]] = {	×
52	status: [],
53	status_esr: [],
54	}
55
56	def handler(bug):	×
57	for field in bug.keys():	×
58	if field.startswith(status_esr):	×
59	data[status_esr].append(field)	×
60	elif field.startswith(status):	×
61	data[status].append(field)	×
62
63	Bugzilla(bugids=1234567, include_fields="_custom", bughandler=handler).wait()	×
64
65	return (	×
66	# ESR
67	sorted(
68	data[status_esr],
69	key=lambda flag: int(flag[len(status_esr) :]),
70	)[-1:]
71	# Release, Beta, Nightly
72	+ sorted(
73	data[status],
74	key=lambda flag: int(flag[len(status) :]),
75	)[-3:]
76	)
77
78	@cached_property	×
79	def nightly_status_flag(self) -> str:	×
80	"""The nightly release status flag for Firefox."""
81	return self.current_status_flags[-1]	×
82
83	def _active_regression_authors(	×
84	self, signatures: list[SignatureAnalyzer]
85	) -> set[str]:
86	"""Get Bugzilla usernames for users who are active and can be needinfo'd.
87
88	Args:
89	signatures: a list of signatures for which to check the status of
90	their regression author.
91
92	Returns:
93	A set of user emails.
94	"""
95	ni_skiplist = self.get_auto_ni_skiplist()	×
96	users = UserActivity(include_fields=["requests"]).check_users(	×
97	(
98	signature.regressed_by_author["name"]
99	for signature in signatures
100	if signature.regressed_by_author
101	),
102	keep_active=True,
103	fetch_employee_info=True,
104	)
105
106	return {	×
107	name
108	for name, user in users.items()
109	if name not in ni_skiplist
110	and user["status"] == UserStatus.ACTIVE
111	and not user["requests"]["needinfo"]["blocked"]
112	}
113
114	def get_bugs(self, date):	×
115	self.query_url = None	×
116	bugs = {}	×
117
118	data_fetcher = SignaturesDataFetcher.find_new_actionable_crashes(	×
119	"Firefox", "nightly"
120	)
121	signatures = data_fetcher.analyze()	×
122	# This is the last filtering stage which aims to avoid filing bugs for
123	# junky crashes, where the volume is low and the crashes do not show
124	# signals of being actionable or critical.
125	signatures = [	×
126	signature
127	for signature in signatures
128	if signature.num_installs > 5
129	or signature.num_crashes > 25
130	or signature.is_potential_near_null_crash
131	or signature.is_potential_security_crash
132	or signature.regressed_by
133	or signature.has_moz_crash_reason("MOZ_RELEASE_ASSERT")
134	or signature.has_moz_crash_reason("DocumentChannel::SetLoadFlags")
135	]
136
137	signature_details_delta = humanize.naturaldelta(data_fetcher.SUMMARY_DURATION)	×
138
139	active_regression_authors = self._active_regression_authors(signatures)	×
140
141	for signature in signatures:	×
142	logger.debug("Generating bug for signature: %s", signature.signature_term)	×
143
144	title = (	×
145	f"Startup crash in [@ {signature.signature_term}]"
146	if signature.is_startup_related_crash
147	else f"Crash in [@ {signature.signature_term}]"
148	)
149	if len(title) > self.MAX_BUG_TITLE_LENGTH:	×
150	title = title[: self.MAX_BUG_TITLE_LENGTH - 3] + "..."	×
151
152	# Whether we should needinfo the regression author.
153	needinfo_regression_author = (	×
154	signature.regressed_by
155	and signature.regressed_by_author["email"] in active_regression_authors
156	)
157
158	report = signature.fetch_representative_processed_crash()	×
159	description = self.bug_description_template.render(	×
160	{
161	**socorro_util.generate_bug_description_data(report),
162	"signature": signature,
163	"needinfo_regression_author": needinfo_regression_author,
164	"signature_details_delta": signature_details_delta,
165	"signature_details_channel": "Nightly",
166	}
167	)
168
169	# TODO: Provide the following information:
170	# [X] Crash signature
171	# [X] Top 10 frames of crashing thread
172	# [X] Component
173	# [X] The kind of crash
174	# [ ] Regression window
175	# [X] Inducing patch
176	# [X] Reason
177	# [X] Regressed by
178	# [X] Platform
179	# [x] Firefox status flags
180	# [ ] Severity
181	# [ ] Time correlation
182	# [X] User comments
183	# [X] Crash address commonalities
184	# [ ] Estimated future crash volume
185
186	bug_data = {	×
187	"blocks": "bugbot-auto-crash",
188	"type": "defect",
189	"keywords": ["crash"],
190	"summary": title,
191	"product": signature.crash_component.product,
192	"component": signature.crash_component.name,
193	"op_sys": signature.bugzilla_op_sys,
194	"rep_platform": signature.bugzilla_cpu_arch,
195	"cf_crash_signature": f"[@ {signature.signature_term}]",
196	"description": description,
197	self.nightly_status_flag: "affected",
198	# NOTE(suhaib): the following CC is for testing purposes only
199	# to allow us access and evaluate security bugs. It should be
200	# removed at some point after we move to production.
201	"cc": [
202	"smujahid@mozilla.com",
203	"mcastelluccio@mozilla.com",
204	"aryx.bugmail@gmx-topmail.de",
205	],
206	}
207
208	if needinfo_regression_author:	×
209	bug_data["flags"] = [	×
210	{
211	"name": "needinfo",
212	"requestee": signature.regressed_by_author["name"],
213	"status": "?",
214	"new": "true",
215	}
216	]
217	bug_data["cc"].append(signature.regressed_by_author["name"])	×
218
219	if signature.is_potential_security_crash:	×
220	bug_data["groups"] = ["core-security"]	×
221
222	if signature.regressed_by:	×
223	bug_data["keywords"].append("regression")	×
224	bug_data["regressed_by"] = [signature.regressed_by]	×
225
226	# Empty statuses are needed to detect the affected releases.
227	for flag in self.current_status_flags:	×
228	if flag not in bug_data:	×
229	bug_data[flag] = "---"	×
230
231	bug_analyzer = BugAnalyzer(bug_data, signature.bugs_store)	×
232	updates = bug_analyzer.detect_version_status_updates()	×
233	for update in updates:	×
234	bug_data[update.flag] = update.status	×
235
236	if self.dryrun:	×
237	logger.info("Dry-run bug:")	×
238	pprint.pprint(bug_data)	×
239	bug_id = str(len(bugs) + 1)	×
240	else:
241	try:	×
242	bug = utils.create_bug(bug_data)	×
243	except requests.HTTPError as err:	×
244	logger.exception(	×
245	"Failed to create a bug for signature %s: %s",
246	signature.signature_term,
247	err.response.text,
248	)
249	continue	×
250
251	bug_id = str(bug["id"])	×
252	# TODO: log the created bugs info somewhere (e.g., DB,
253	# spreadsheet, or LabelStudio)
254
255	bugs[bug_id] = {	×
256	"id": bug_id,
257	"summary": "..." if signature.is_potential_security_crash else title,
258	"component": signature.crash_component,
259	}
260
261	logger.debug("Total of %d bugs have been filed", len(bugs))	×
262
263	return bugs	×
264
265
266	if __name__ == "__main__":	×
267	FileCrashBug().run()	×

mozilla / relman-auto-nag / #4841

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous