• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

mozilla / relman-auto-nag / #5158

12 Jul 2024 12:25AM CUT coverage: 21.768% (+0.2%) from 21.54%
#5158

push

coveralls-python

suhaibmujahid
[component ] Skip bugs that are not available for BugBug

585 of 3497 branches covered (16.73%)

0 of 2 new or added lines in 1 file covered. (0.0%)

317 existing lines in 5 files now uncovered.

1933 of 8880 relevant lines covered (21.77%)

0.22 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0
/bugbot/rules/component.py
1
# This Source Code Form is subject to the terms of the Mozilla Public
2
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
3
# You can obtain one at http://mozilla.org/MPL/2.0/.
4

5
from libmozdata.bugzilla import Bugzilla
×
6

7
from bugbot import logger
×
8
from bugbot.bugbug_utils import get_bug_ids_classification
×
9
from bugbot.bzcleaner import BzCleaner
×
10
from bugbot.utils import get_config, nice_round
×
11

12

13
class Component(BzCleaner):
×
14
    def __init__(self):
×
15
        super().__init__()
×
16
        self.autofix_component = {}
×
17
        self.frequency = "daily"
×
UNCOV
18
        self.general_confidence_threshold = self.get_config(
×
19
            "general_confidence_threshold"
20
        )
UNCOV
21
        self.component_confidence_threshold = self.get_config("confidence_threshold")
×
UNCOV
22
        self.fenix_confidence_threshold = self.get_config("fenix_confidence_threshold")
×
23

UNCOV
24
    def add_custom_arguments(self, parser):
×
UNCOV
25
        parser.add_argument(
×
26
            "--frequency",
27
            help="Daily (noisy) or Hourly",
28
            choices=["daily", "hourly"],
29
            default="daily",
30
        )
31

UNCOV
32
    def parse_custom_arguments(self, args):
×
33
        self.frequency = args.frequency
×
34

UNCOV
35
    def description(self):
×
36
        return f"[Using ML] Assign a component to untriaged bugs ({self.frequency})"
×
37

UNCOV
38
    def columns(self):
×
39
        return ["id", "summary", "component", "confidence", "autofixed"]
×
40

41
    def sort_columns(self):
×
UNCOV
42
        return lambda p: (-p[3], -int(p[0]))
×
43

44
    def has_product_component(self):
×
45
        # Inject product and components when calling BzCleaner.get_bugs
46
        return True
×
47

48
    def get_bz_params(self, date):
×
UNCOV
49
        start_date, end_date = self.get_dates(date)
×
50

UNCOV
51
        bot = get_config("common", "bot_bz_mail")[0]
×
52

UNCOV
53
        return {
×
54
            "include_fields": ["id", "groups", "summary", "product", "component"],
55
            # Ignore bugs for which we ever modified the product or the component.
56
            "n1": 1,
57
            "f1": "product",
58
            "o1": "changedby",
59
            "v1": bot,
60
            "n2": 1,
61
            "f2": "component",
62
            "o2": "changedby",
63
            "v2": bot,
64
            # Ignore closed bugs.
65
            "bug_status": "__open__",
66
            # Get recent General bugs, and all Untriaged bugs.
67
            "j3": "OR",
68
            "f3": "OP",
69
            "j4": "AND",
70
            "f4": "OP",
71
            "f5": "component",
72
            "o5": "equals",
73
            "v5": "General",
74
            "f6": "creation_ts",
75
            "o6": "greaterthan",
76
            "v6": start_date,
77
            "f7": "CP",
78
            "f8": "component",
79
            "o8": "anyexact",
80
            "v8": "Untriaged,Foxfooding",
81
            "f9": "CP",
82
        }
83

84
    def get_bugs(self, date="today", bug_ids=[]):
×
UNCOV
85
        def meets_threshold(bug_data):
×
UNCOV
86
            threshold = (
×
87
                self.general_confidence_threshold
88
                if bug_data["class"] == "Fenix" or bug_data["class"] == "General"
89
                else self.component_confidence_threshold
90
            )
UNCOV
91
            return bug_data["prob"][bug_data["index"]] >= threshold
×
92

93
        # Retrieve the bugs with the fields defined in get_bz_params
94
        raw_bugs = super().get_bugs(date=date, bug_ids=bug_ids, chunk_size=7000)
×
95

UNCOV
96
        if len(raw_bugs) == 0:
×
97
            return {}
×
98

99
        # Extract the bug ids
100
        bug_ids = list(raw_bugs.keys())
×
101

102
        # Classify those bugs
103
        bugs = get_bug_ids_classification("component", bug_ids)
×
104

105
        fenix_general_bug_ids = []
×
106
        for bug_id, bug_data in bugs.items():
×
NEW
107
            if not bug_data.get("available", True):
×
108
                # The bug was not available, it was either removed or is a
109
                # security bug.
NEW
110
                continue
×
111
            if meets_threshold(bug_data):
×
112
                if bug_data.get("class") == "Fenix":
×
113
                    fenix_general_bug_ids.append(bug_id)
×
114
            else:
UNCOV
115
                if (
×
116
                    raw_bugs[bug_id]["product"] == "Fenix"
117
                    and raw_bugs[bug_id]["component"] == "General"
118
                ):
119
                    fenix_general_bug_ids.append(bug_id)
×
120

121
        if fenix_general_bug_ids:
×
UNCOV
122
            fenix_general_classification = get_bug_ids_classification(
×
123
                "fenixcomponent", fenix_general_bug_ids
124
            )
125

UNCOV
126
            for bug_id, data in fenix_general_classification.items():
×
127
                confidence = data["prob"][data["index"]]
×
128

129
                if confidence > self.fenix_confidence_threshold:
×
130
                    data["class"] = f"Fenix::{data['class']}"
×
131
                    bugs[bug_id] = data
×
132

UNCOV
133
        results = {}
×
134

UNCOV
135
        for bug_id in sorted(bugs.keys()):
×
UNCOV
136
            bug_data = bugs[bug_id]
×
137

138
            if not bug_data.get("available", True):
×
139
                # The bug was not available, it was either removed or is a
140
                # security bug
141
                continue
×
142

UNCOV
143
            if not {"prob", "index", "class", "extra_data"}.issubset(bug_data.keys()):
×
UNCOV
144
                raise Exception(f"Invalid bug response {bug_id}: {bug_data!r}")
×
145

UNCOV
146
            bug = raw_bugs[bug_id]
×
147
            prob = bug_data["prob"]
×
UNCOV
148
            index = bug_data["index"]
×
149
            suggestion = bug_data["class"]
×
150

UNCOV
151
            conflated_components_mapping = bug_data["extra_data"].get(
×
152
                "conflated_components_mapping", {}
153
            )
154

155
            # Skip product-only suggestions that are not useful.
UNCOV
156
            if "::" not in suggestion and bug["product"] == suggestion:
×
UNCOV
157
                continue
×
158

159
            suggestion = conflated_components_mapping.get(suggestion, suggestion)
×
160

161
            if "::" not in suggestion:
×
UNCOV
162
                logger.error(
×
163
                    f"There is something wrong with this component suggestion! {suggestion}"
164
                )
UNCOV
165
                continue
×
166

167
            i = suggestion.index("::")
×
168
            suggested_product = suggestion[:i]
×
UNCOV
169
            suggested_component = suggestion[i + 2 :]
×
170

171
            # When moving bugs out of the 'General' component, we don't want to change the product (unless it is Firefox).
UNCOV
172
            if bug["component"] == "General" and bug["product"] not in {
×
173
                suggested_product,
174
                "Firefox",
175
            }:
176
                continue
×
177

178
            # Don't move bugs from Firefox::General to Core::Internationalization.
UNCOV
179
            if (
×
180
                bug["product"] == "Firefox"
181
                and bug["component"] == "General"
182
                and suggested_product == "Core"
183
                and suggested_component == "Internationalization"
184
            ):
UNCOV
185
                continue
×
186

187
            result = {
×
188
                "id": bug_id,
189
                "summary": bug["summary"],
190
                "component": suggestion,
191
                "confidence": nice_round(prob[index]),
192
                "autofixed": False,
193
            }
194

195
            # In daily mode, we send an email with all results.
196
            if self.frequency == "daily":
×
UNCOV
197
                results[bug_id] = result
×
198

199
            confidence_threshold_conf = (
×
200
                "confidence_threshold"
201
                if bug["component"] != "General"
202
                else "general_confidence_threshold"
203
            )
204

205
            if prob[index] >= self.get_config(confidence_threshold_conf):
×
206
                self.autofix_component[bug_id] = {
×
207
                    "product": suggested_product,
208
                    "component": suggested_component,
209
                }
210

211
                result["autofixed"] = True
×
212

213
                # In hourly mode, we send an email with only the bugs we acted upon.
214
                if self.frequency == "hourly":
×
215
                    results[bug_id] = result
×
216

217
        # Don't move bugs back into components they were moved out of.
218
        # TODO: Use the component suggestion from the service with the second highest confidence instead.
UNCOV
219
        def history_handler(bug):
×
UNCOV
220
            bug_id = str(bug["id"])
×
221

UNCOV
222
            previous_product_components = set()
×
223

UNCOV
224
            current_product = raw_bugs[bug_id]["product"]
×
UNCOV
225
            current_component = raw_bugs[bug_id]["component"]
×
226

UNCOV
227
            for history in bug["history"]:
×
UNCOV
228
                for change in history["changes"][::-1]:
×
UNCOV
229
                    if change["field_name"] == "product":
×
230
                        current_product = change["removed"]
×
231
                    elif change["field_name"] == "component":
×
UNCOV
232
                        current_component = change["removed"]
×
233

UNCOV
234
                previous_product_components.add((current_product, current_component))
×
235

UNCOV
236
            suggested_product = self.autofix_component[bug_id]["product"]
×
237
            suggested_component = self.autofix_component[bug_id]["component"]
×
238

UNCOV
239
            if (suggested_product, suggested_component) in previous_product_components:
×
UNCOV
240
                results[bug_id]["autofixed"] = False
×
UNCOV
241
                del self.autofix_component[bug_id]
×
242

UNCOV
243
        bugids = list(self.autofix_component.keys())
×
UNCOV
244
        Bugzilla(
×
245
            bugids=bugids,
246
            historyhandler=history_handler,
247
        ).get_data().wait()
248

UNCOV
249
        return results
×
250

UNCOV
251
    def get_autofix_change(self):
×
UNCOV
252
        cc = self.get_config("cc")
×
UNCOV
253
        return {
×
254
            bug_id: (
255
                data.update(
256
                    {
257
                        "cc": {"add": cc},
258
                        "comment": {
259
                            "body": f"The [Bugbug](https://github.com/mozilla/bugbug/) bot thinks this bug should belong to the '{data['product']}::{data['component']}' component, and is moving the bug to that component. Please correct in case you think the bot is wrong."
260
                        },
261
                    }
262
                )
263
                or data
264
            )
265
            for bug_id, data in self.autofix_component.items()
266
        }
267

UNCOV
268
    def get_db_extra(self):
×
UNCOV
269
        return {
×
270
            bugid: "{}::{}".format(v["product"], v["component"])
271
            for bugid, v in self.get_autofix_change().items()
272
        }
273

274

UNCOV
275
if __name__ == "__main__":
×
UNCOV
276
    Component().run()
×
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2025 Coveralls, Inc