#5011

Committed 17 May 2024 05:04PM UTC coverage: 21.84% (+0.04%) from 21.805%

Build # #5011

Build Type

push

coveralls-python

Committed by

benjaminmah

Commit Message

Removed duplicate code.

Run Details

716 of 3598 branches covered (19.9%)

0 of 39 new or added lines in 1 file covered. (0.0%)

1 existing line in 1 file now uncovered.

1930 of 8837 relevant lines covered (21.84%)

0.22 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/bugbot/rules/component.py

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.

from libmozdata.bugzilla import Bugzilla

from bugbot import logger
from bugbot.bugbug_utils import get_bug_ids_classification
from bugbot.bzcleaner import BzCleaner
from bugbot.utils import get_config, nice_round


class Component(BzCleaner):
    def __init__(self):
        super().__init__()
        self.autofix_component = {}
        self.frequency = "daily"

    def add_custom_arguments(self, parser):
        parser.add_argument(
            "--frequency",
            help="Daily (noisy) or Hourly",
            choices=["daily", "hourly"],
            default="daily",
        )

    def parse_custom_arguments(self, args):
        self.frequency = args.frequency

    def description(self):
        return f"[Using ML] Assign a component to untriaged bugs ({self.frequency})"

    def columns(self):
        return ["id", "summary", "component", "confidence", "autofixed"]

    def sort_columns(self):
        return lambda p: (-p[3], -int(p[0]))

    def has_product_component(self):
        # Inject product and components when calling BzCleaner.get_bugs
        return True

    def get_bz_params(self, date):
        start_date, end_date = self.get_dates(date)

        bot = get_config("common", "bot_bz_mail")[0]

        return {
            "include_fields": ["id", "groups", "summary", "product", "component"],
            # Ignore bugs for which we ever modified the product or the component.
            "n1": 1,
            "f1": "product",
            "o1": "changedby",
            "v1": bot,
            "n2": 1,
            "f2": "component",
            "o2": "changedby",
            "v2": bot,
            # Ignore closed bugs.
            "bug_status": "__open__",
            # Get recent General bugs, and all Untriaged bugs.
            "j3": "OR",
            "f3": "OP",
            "j4": "AND",
            "f4": "OP",
            "f5": "component",
            "o5": "equals",
            "v5": "General",
            "f6": "creation_ts",
            "o6": "greaterthan",
            "v6": start_date,
            "f7": "CP",
            "f8": "component",
            "o8": "anyexact",
            "v8": "Untriaged,Foxfooding",
            "f9": "CP",
        }

    def get_bugs(self, date="today", bug_ids=[]):
        def process_bugs(bugs, raw_bugs, results, reclassify_fenix):
            for bug_id in sorted(bugs.keys()):
                bug_data = bugs[bug_id]

                if not bug_data.get("available", True):
                    # The bug was not available, it was either removed or is a
                    # security bug
                    continue

                if not {"prob", "index", "class", "extra_data"}.issubset(
                    bug_data.keys()
                ):
                    raise Exception(f"Invalid bug response {bug_id}: {bug_data!r}")

                bug = raw_bugs[bug_id]
                prob = bug_data["prob"]
                index = bug_data["index"]
                suggestion = bug_data["class"]
                conflated_components_mapping = bug_data["extra_data"][
                    "conflated_components_mapping"
                ]

                # Skip product-only suggestions that are not useful.
                if "::" not in suggestion and bug["product"] == suggestion:
                    continue

                suggestion = conflated_components_mapping.get(suggestion, suggestion)

                if "::" not in suggestion:
                    logger.error(
                        f"There is something wrong with this component suggestion! {suggestion}"
                    )
                    continue

                i = suggestion.index("::")
                suggested_product = suggestion[:i]
                suggested_component = suggestion[i + 2 :]

                # When moving bugs out of the 'General' component, we don't want to change the product (unless it is Firefox).
                if bug["component"] == "General" and bug["product"] not in {
                    suggested_product,
                    "Firefox",
                }:
                    continue

                # Don't move bugs from Firefox::General to Core::Internationalization.
                if (
                    bug["product"] == "Firefox"
                    and bug["component"] == "General"
                    and suggested_product == "Core"
                    and suggested_component == "Internationalization"
                ):
                    continue

                result = {
                    "id": bug_id,
                    "summary": bug["summary"],
                    "component": suggestion,
                    "confidence": nice_round(prob[index]),
                    "autofixed": False,
                }

                # In daily mode, we send an email with all results.
                if self.frequency == "daily":
                    results[bug_id] = result

                confidence_threshold_conf = (
                    "confidence_threshold"
                    if bug["component"] != "General"
                    else "general_confidence_threshold"
                )

                if prob[index] >= self.get_config(confidence_threshold_conf):
                    self.autofix_component[bug_id] = {
                        "product": suggested_product,
                        "component": suggested_component,
                    }

                    result["autofixed"] = True

                    # In hourly mode, we send an email with only the bugs we acted upon.
                    if self.frequency == "hourly":
                        results[bug_id] = result

                # Collect bugs that were classified as Fenix::General
                if bug["product"] == "Fenix" and bug["component"] == "General":
                    fenix_general_bug_ids.append(bug_id)

            return fenix_general_bug_ids

        # Retrieve the bugs with the fields defined in get_bz_params
        raw_bugs = super().get_bugs(date=date, bug_ids=bug_ids, chunk_size=7000)

        if len(raw_bugs) == 0:
            return {}

        # Extract the bug ids
        bug_ids = list(raw_bugs.keys())

        # Classify those bugs
        bugs = get_bug_ids_classification("component", bug_ids)

        results = {}

        # List to store IDs of bugs classified as Fenix::General
        fenix_general_bug_ids = []

        fenix_general_bug_ids = process_bugs(
            bugs, raw_bugs, results, fenix_general_bug_ids
        )

        if fenix_general_bug_ids:
            fenix_bugs = get_bug_ids_classification(
                "fenixcomponent", fenix_general_bug_ids
            )
            process_bugs(fenix_bugs, raw_bugs, results)

        # Don't move bugs back into components they were moved out of.
        # TODO: Use the component suggestion from the service with the second highest confidence instead.
        def history_handler(bug):
            bug_id = str(bug["id"])

            previous_product_components = set()

            current_product = raw_bugs[bug_id]["product"]
            current_component = raw_bugs[bug_id]["component"]

            for history in bug["history"]:
                for change in history["changes"][::-1]:
                    if change["field_name"] == "product":
                        current_product = change["removed"]
                    elif change["field_name"] == "component":
                        current_component = change["removed"]

                previous_product_components.add((current_product, current_component))

            suggested_product = self.autofix_component[bug_id]["product"]
            suggested_component = self.autofix_component[bug_id]["component"]

            if (suggested_product, suggested_component) in previous_product_components:
                results[bug_id]["autofixed"] = False
                del self.autofix_component[bug_id]

        bugids = list(self.autofix_component.keys())
        Bugzilla(
            bugids=bugids,
            historyhandler=history_handler,
        ).get_data().wait()

        return results

    def get_autofix_change(self):
        cc = self.get_config("cc")
        return {
            bug_id: (
                data.update(
                    {
                        "cc": {"add": cc},
                        "comment": {
                            "body": f"The [Bugbug](https://github.com/mozilla/bugbug/) bot thinks this bug should belong to the '{data['product']}::{data['component']}' component, and is moving the bug to that component. Please correct in case you think the bot is wrong."
                        },
                    }
                )
                or data
            )
            for bug_id, data in self.autofix_component.items()
        }

    def get_db_extra(self):
        return {
            bugid: "{}::{}".format(v["product"], v["component"])
            for bugid, v in self.get_autofix_change().items()
        }


if __name__ == "__main__":
    Component().run()

1	# This Source Code Form is subject to the terms of the Mozilla Public
2	# License, v. 2.0. If a copy of the MPL was not distributed with this file,
3	# You can obtain one at http://mozilla.org/MPL/2.0/.
4
5	from libmozdata.bugzilla import Bugzilla	×
6
7	from bugbot import logger	×
8	from bugbot.bugbug_utils import get_bug_ids_classification	×
9	from bugbot.bzcleaner import BzCleaner	×
10	from bugbot.utils import get_config, nice_round	×
11
12
13	class Component(BzCleaner):	×
14	def __init__(self):	×
15	super().__init__()	×
16	self.autofix_component = {}	×
17	self.frequency = "daily"	×
18
19	def add_custom_arguments(self, parser):	×
20	parser.add_argument(	×
21	"--frequency",
22	help="Daily (noisy) or Hourly",
23	choices=["daily", "hourly"],
24	default="daily",
25	)
26
27	def parse_custom_arguments(self, args):	×
28	self.frequency = args.frequency	×
29
30	def description(self):	×
31	return f"[Using ML] Assign a component to untriaged bugs ({self.frequency})"	×
32
33	def columns(self):	×
34	return ["id", "summary", "component", "confidence", "autofixed"]	×
35
36	def sort_columns(self):	×
37	return lambda p: (-p[3], -int(p[0]))	×
38
39	def has_product_component(self):	×
40	# Inject product and components when calling BzCleaner.get_bugs
41	return True	×
42
43	def get_bz_params(self, date):	×
44	start_date, end_date = self.get_dates(date)	×
45
46	bot = get_config("common", "bot_bz_mail")[0]	×
47
48	return {	×
49	"include_fields": ["id", "groups", "summary", "product", "component"],
50	# Ignore bugs for which we ever modified the product or the component.
51	"n1": 1,
52	"f1": "product",
53	"o1": "changedby",
54	"v1": bot,
55	"n2": 1,
56	"f2": "component",
57	"o2": "changedby",
58	"v2": bot,
59	# Ignore closed bugs.
60	"bug_status": "__open__",
61	# Get recent General bugs, and all Untriaged bugs.
62	"j3": "OR",
63	"f3": "OP",
64	"j4": "AND",
65	"f4": "OP",
66	"f5": "component",
67	"o5": "equals",
68	"v5": "General",
69	"f6": "creation_ts",
70	"o6": "greaterthan",
71	"v6": start_date,
72	"f7": "CP",
73	"f8": "component",
74	"o8": "anyexact",
75	"v8": "Untriaged,Foxfooding",
76	"f9": "CP",
77	}
78
79	def get_bugs(self, date="today", bug_ids=[]):	×
NEW 80	def process_bugs(bugs, raw_bugs, results, reclassify_fenix):	×
NEW 81	for bug_id in sorted(bugs.keys()):	×
NEW 82	bug_data = bugs[bug_id]	×
83
NEW 84	if not bug_data.get("available", True):	×
85	# The bug was not available, it was either removed or is a
86	# security bug
NEW 87	continue	×
88
NEW 89	if not {"prob", "index", "class", "extra_data"}.issubset(	×
90	bug_data.keys()
91	):
NEW 92	raise Exception(f"Invalid bug response {bug_id}: {bug_data!r}")	×
93
NEW 94	bug = raw_bugs[bug_id]	×
NEW 95	prob = bug_data["prob"]	×
NEW 96	index = bug_data["index"]	×
NEW 97	suggestion = bug_data["class"]	×
NEW 98	conflated_components_mapping = bug_data["extra_data"][	×
99	"conflated_components_mapping"
100	]
101
102	# Skip product-only suggestions that are not useful.
NEW 103	if "::" not in suggestion and bug["product"] == suggestion:	×
NEW 104	continue	×
105
NEW 106	suggestion = conflated_components_mapping.get(suggestion, suggestion)	×
107
NEW 108	if "::" not in suggestion:	×
NEW 109	logger.error(	×
110	f"There is something wrong with this component suggestion! {suggestion}"
111	)
NEW 112	continue	×
113
114	i = suggestion.index("::")	×
115	suggested_product = suggestion[:i]	×
116	suggested_component = suggestion[i + 2 :]	×
117
118	# When moving bugs out of the 'General' component, we don't want to change the product (unless it is Firefox).
NEW 119	if bug["component"] == "General" and bug["product"] not in {	×
120	suggested_product,
121	"Firefox",
122	}:
NEW 123	continue	×
124
125	# Don't move bugs from Firefox::General to Core::Internationalization.
NEW 126	if (	×
127	bug["product"] == "Firefox"
128	and bug["component"] == "General"
129	and suggested_product == "Core"
130	and suggested_component == "Internationalization"
131	):
NEW 132	continue	×
133
UNCOV 134	result = {	×
135	"id": bug_id,
136	"summary": bug["summary"],
137	"component": suggestion,
138	"confidence": nice_round(prob[index]),
139	"autofixed": False,
140	}
141
142	# In daily mode, we send an email with all results.
143	if self.frequency == "daily":	×
NEW 144	results[bug_id] = result	×
145
146	confidence_threshold_conf = (	×
147	"confidence_threshold"
148	if bug["component"] != "General"
149	else "general_confidence_threshold"
150	)
151
152	if prob[index] >= self.get_config(confidence_threshold_conf):	×
NEW 153	self.autofix_component[bug_id] = {	×
154	"product": suggested_product,
155	"component": suggested_component,
156	}
157
158	result["autofixed"] = True	×
159
160	# In hourly mode, we send an email with only the bugs we acted upon.
161	if self.frequency == "hourly":	×
NEW 162	results[bug_id] = result	×
163
164	# Collect bugs that were classified as Fenix::General
NEW 165	if bug["product"] == "Fenix" and bug["component"] == "General":	×
NEW 166	fenix_general_bug_ids.append(bug_id)	×
167
NEW 168	return fenix_general_bug_ids	×
169
170	# Retrieve the bugs with the fields defined in get_bz_params
NEW 171	raw_bugs = super().get_bugs(date=date, bug_ids=bug_ids, chunk_size=7000)	×
172
NEW 173	if len(raw_bugs) == 0:	×
NEW 174	return {}	×
175
176	# Extract the bug ids
NEW 177	bug_ids = list(raw_bugs.keys())	×
178
179	# Classify those bugs
NEW 180	bugs = get_bug_ids_classification("component", bug_ids)	×
181
NEW 182	results = {}	×
183
184	# List to store IDs of bugs classified as Fenix::General
NEW 185	fenix_general_bug_ids = []	×
186
NEW 187	fenix_general_bug_ids = process_bugs(	×
188	bugs, raw_bugs, results, fenix_general_bug_ids
189	)
190
NEW 191	if fenix_general_bug_ids:	×
NEW 192	fenix_bugs = get_bug_ids_classification(	×
193	"fenixcomponent", fenix_general_bug_ids
194	)
NEW 195	process_bugs(fenix_bugs, raw_bugs, results)	×
196
197	# Don't move bugs back into components they were moved out of.
198	# TODO: Use the component suggestion from the service with the second highest confidence instead.
199	def history_handler(bug):	×
200	bug_id = str(bug["id"])	×
201
202	previous_product_components = set()	×
203
204	current_product = raw_bugs[bug_id]["product"]	×
205	current_component = raw_bugs[bug_id]["component"]	×
206
207	for history in bug["history"]:	×
208	for change in history["changes"][::-1]:	×
209	if change["field_name"] == "product":	×
210	current_product = change["removed"]	×
211	elif change["field_name"] == "component":	×
212	current_component = change["removed"]	×
213
214	previous_product_components.add((current_product, current_component))	×
215
216	suggested_product = self.autofix_component[bug_id]["product"]	×
217	suggested_component = self.autofix_component[bug_id]["component"]	×
218
219	if (suggested_product, suggested_component) in previous_product_components:	×
220	results[bug_id]["autofixed"] = False	×
221	del self.autofix_component[bug_id]	×
222
223	bugids = list(self.autofix_component.keys())	×
224	Bugzilla(	×
225	bugids=bugids,
226	historyhandler=history_handler,
227	).get_data().wait()
228
229	return results	×
230
231	def get_autofix_change(self):	×
232	cc = self.get_config("cc")	×
233	return {	×
234	bug_id: (
235	data.update(
236	{
237	"cc": {"add": cc},
238	"comment": {
239	"body": f"The [Bugbug](https://github.com/mozilla/bugbug/) bot thinks this bug should belong to the '{data['product']}::{data['component']}' component, and is moving the bug to that component. Please correct in case you think the bot is wrong."
240	},
241	}
242	)
243	or data
244	)
245	for bug_id, data in self.autofix_component.items()
246	}
247
248	def get_db_extra(self):	×
249	return {	×
250	bugid: "{}::{}".format(v["product"], v["component"])
251	for bugid, v in self.get_autofix_change().items()
252	}
253
254
255	if __name__ == "__main__":	×
256	Component().run()	×

mozilla / relman-auto-nag / #5011

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous