#5167

Committed 17 Jul 2024 02:23PM CUT coverage: 21.862% (+0.005%) from 21.857%

Build # #5167

Build Type

push

coveralls-python

Committed by

benjaminmah

Commit Message

Removed `nightly_pat` check and moved backout check to be after handling comments

Run Details

716 of 3604 branches covered (19.87%)

0 of 6 new or added lines in 1 file covered. (0.0%)

2 existing lines in 1 file now uncovered.

1933 of 8842 relevant lines covered (21.86%)

0.22 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/bugbot/rules/not_landed.py

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.

import base64
import random
import re

from libmozdata import utils as lmdutils
from libmozdata.bugzilla import Bugzilla, BugzillaUser
from libmozdata.phabricator import (
    PhabricatorAPI,
    PhabricatorBzNotFoundException,
    PhabricatorRevisionNotFoundException,
)

from bugbot import utils
from bugbot.bzcleaner import BzCleaner

PHAB_URL_PAT = re.compile(r"https://phabricator\.services\.mozilla\.com/D([0-9]+)")


class NotLanded(BzCleaner):
    def __init__(self):
        super(NotLanded, self).__init__()
        self.nweeks = utils.get_config(self.name(), "number_of_weeks", 2)
        self.nyears = utils.get_config(self.name(), "number_of_years", 2)
        self.phab = PhabricatorAPI(utils.get_login_info()["phab_api_key"])
        self.extra_ni = {}

    def description(self):
        return "Open bugs with no activity for {} weeks and a r+ patch which hasn't landed".format(
            self.nweeks
        )

    def has_assignee(self):
        return True

    def get_extra_for_template(self):
        return {"nweeks": self.nweeks}

    def get_extra_for_needinfo_template(self):
        self.extra_ni.update(self.get_extra_for_template())
        return self.extra_ni

    def columns(self):
        return ["id", "summary", "assignee"]

    def handle_bug(self, bug, data):
        if self.has_bot_set_ni(bug):
            return None

        bugid = str(bug["id"])
        assignee = bug.get("assigned_to", "")
        if utils.is_no_assignee(assignee):
            assignee = ""
            nickname = ""
        else:
            nickname = bug["assigned_to_detail"]["nick"]

        data[bugid] = {
            "assigned_to": assignee,
            "nickname": nickname,
            "deps": set(bug["depends_on"]),
        }

        return bug

    def update_revision_status(self, rev_id, status_type, value):
        try:
            self.phab.request(
                "differential.revision.edit",
                objectIdentifier=rev_id,
                transactions=[{"type": status_type, "value": value}],
            )
            print(
                f"Successfully updated revision {rev_id} with {status_type} to {value}"
            )
        except Exception as e:
            print(
                f"Failed to update revision {rev_id} with {status_type} to {value}: {e}"
            )

    def filter_bugs(self, bugs):
        # We must remove bugs which have open dependencies (except meta bugs)
        # because devs may wait for those bugs to be fixed before their patch
        # can land.

        all_deps = set(dep for info in bugs.values() for dep in info["deps"])

        def bug_handler(bug, data):
            if (
                bug["status"] in {"RESOLVED", "VERIFIED", "CLOSED"}
                or "meta" in bug["keywords"]
            ):
                data.add(bug["id"])

        useless = set()
        Bugzilla(
            bugids=list(all_deps),
            include_fields=["id", "keywords", "status"],
            bughandler=bug_handler,
            bugdata=useless,
        ).get_data().wait()

        for bugid, info in bugs.items():
            # finally deps will contain open bugs which are not meta
            info["deps"] -= useless

        # keep bugs with no deps
        bugs = {bugid: info for bugid, info in bugs.items() if not info["deps"]}

        return bugs

    def check_phab(self, attachment, reviewers_phid):
        """Check if the patch in Phabricator has been r+"""
        if attachment["is_obsolete"] == 1:
            return None

        phab_url = base64.b64decode(attachment["data"]).decode("utf-8")

        # extract the revision
        rev = PHAB_URL_PAT.search(phab_url).group(1)
        try:
            data = self.phab.load_revision(
                rev_id=int(rev), queryKey="all", attachments={"reviewers": 1}
            )
        except PhabricatorRevisionNotFoundException:
            return None

        # this is a timestamp
        last_modified = data["fields"]["dateModified"]
        last_modified = lmdutils.get_date_from_timestamp(last_modified)
        if (self.date - last_modified).days <= self.nweeks * 7:
            # Do not do anything if recent changes in the bug
            return False

        reviewers = data["attachments"]["reviewers"]["reviewers"]

        if not reviewers:
            return False

        for reviewer in reviewers:
            if reviewer["status"] != "accepted":
                return False
            reviewers_phid.add(reviewer["reviewerPHID"])

        value = data["fields"]["status"].get("value", "")
        if value == "changes-planned":
            # even if the patch is r+ and not published, some changes may be required
            # so with the value 'changes-planned', the dev can say it's still a wip
            return False

        if value != "published":
            return True

        return False

    def handle_attachment(self, attachment, res):
        ct = attachment["content_type"]
        c = None
        if ct == "text/x-phabricator-request":
            if "phab" not in res or res["phab"]:
                c = self.check_phab(attachment, res["reviewers_phid"])
                if c is not None:
                    res["phab"] = c

        if c is not None:
            attacher = attachment["creator"]
            if "author" in res:
                if attacher in res["author"]:
                    res["author"][attacher] += 1
                else:
                    res["author"][attacher] = 1
            else:
                res["author"] = {attacher: 1}

            if "count" in res:
                res["count"] += 1
            else:
                res["count"] = 1

    def get_patch_data(self, bugs):
        """Get patch information in bugs"""

        def comment_handler(bug, bugid, data):
            # if a comment contains a backout: don't nag
            for comment in bug["comments"]:
                comment = comment["text"].lower()
                if "backed out" in comment or "backout" in comment:
                    data[bugid]["backout"] = True

        def attachment_id_handler(attachments, bugid, data):
            for a in attachments:
                if (
                    a["content_type"] == "text/x-phabricator-request"
                    and a["is_obsolete"] == 0
                ):
                    data.append(a["id"])

        def attachment_handler(attachments, data):
            for attachment in attachments:
                bugid = str(attachment["bug_id"])
                if bugid in data:
                    data[bugid].append(attachment)
                else:
                    data[bugid] = [attachment]

        bugids = list(bugs.keys())
        data = {
            bugid: {"backout": False, "author": None, "count": 0} for bugid in bugids
        }

        # Get the ids of the attachments of interest
        # to avoid to download images, videos, ...
        attachment_ids = []
        Bugzilla(
            bugids=bugids,
            attachmenthandler=attachment_id_handler,
            attachmentdata=attachment_ids,
            attachment_include_fields=["is_obsolete", "content_type", "id"],
        ).get_data().wait()

        # Once we've the ids we can get the data
        attachments_by_bug = {}
        Bugzilla(
            attachmentids=attachment_ids,
            attachmenthandler=attachment_handler,
            attachmentdata=attachments_by_bug,
            attachment_include_fields=[
                "bug_id",
                "data",
                "is_obsolete",
                "content_type",
                "id",
                "creator",
            ],
        ).get_data().wait()

        for bugid, attachments in attachments_by_bug.items():
            res = {"reviewers_phid": set()}
            for attachment in attachments:
                self.handle_attachment(attachment, res)

            if "phab" in res:
                if res["phab"]:
                    data[bugid]["reviewers_phid"] = res["reviewers_phid"]
                    data[bugid]["author"] = res["author"]
                    data[bugid]["count"] = res["count"]

        data = {bugid: v for bugid, v in data.items() if v["author"]}

        if not data:
            return data

        Bugzilla(
            bugids=list(data.keys()),
            commenthandler=comment_handler,
            commentdata=data,
            comment_include_fields=["text"],
        ).get_data().wait()

        for bugid, v in data.items():
            if data[bugid]["backout"]:
                phab_url = base64.b64decode(attachment["data"]).decode("utf-8")
                rev = PHAB_URL_PAT.search(phab_url).group(1)
                self.update_revision_status(int(rev), "plan-changes", True)

        data = {bugid: v for bugid, v in data.items() if not v["backout"]}

        return data

    def get_bz_userid(self, phids):
        if not phids:
            return {}

        try:
            data = self.phab.load_bz_account(user_phids=list(phids))
            users = {x["phid"]: x["id"] for x in data}
        except PhabricatorBzNotFoundException:
            return {}

        def handler(user, data):
            data[str(user["id"])] = user["name"]

        data = {}
        BugzillaUser(
            user_names=list(users.values()),
            include_fields=["id", "name"],
            user_handler=handler,
            user_data=data,
        ).wait()

        return {phid: data[id] for phid, id in users.items()}

    def get_nicks(self, nicknames):
        def handler(user, data):
            data[user["name"]] = user["nick"]

        users = set(nicknames.values())
        data = {}
        if users:
            BugzillaUser(
                user_names=list(users),
                include_fields=["name", "nick"],
                user_handler=handler,
                user_data=data,
            ).wait()

        for bugid, name in nicknames.items():
            nicknames[bugid] = (name, data[name])

        return nicknames

    def get_bz_params(self, date):
        self.date = lmdutils.get_date_ymd(date)
        fields = ["flags", "depends_on"]
        params = {
            "include_fields": fields,
            "resolution": "---",
            "f1": "attachment.ispatch",
            "n2": 1,
            "f2": "attachments.isobsolete",
            "f3": "attachments.mimetype",
            "o3": "anywordssubstr",
            "v3": "text/x-phabricator-request",
            "f4": "creation_ts",
            "o4": "greaterthan",
            "v4": f"-{self.nyears}y",
            "f5": "days_elapsed",
            "o5": "greaterthaneq",
            "v5": self.nweeks * 7,
            "n6": 1,
            "f6": "longdesc",
            "o6": "casesubstring",
            "v6": "which didn't land and no activity in this bug for",
            "f7": "status_whiteboard",
            "o7": "notsubstring",
            "v7": "[reminder-test ",
        }

        return params

    def get_bugs(self, date="today", bug_ids=[]):
        bugs = super(NotLanded, self).get_bugs(date=date, bug_ids=bug_ids)
        bugs = self.filter_bugs(bugs)
        bugs_patch = self.get_patch_data(bugs)
        res = {}

        reviewers_phid = set()
        nicknames = {}
        for bugid, data in bugs_patch.items():
            reviewers_phid |= data["reviewers_phid"]
            assignee = bugs[bugid]["assigned_to"]
            if not assignee:
                assignee = max(data["author"], key=data["author"].get)
                nicknames[bugid] = assignee

        bz_reviewers = self.get_bz_userid(reviewers_phid)
        all_reviewers = set(bz_reviewers.keys())
        nicknames = self.get_nicks(nicknames)

        for bugid, data in bugs_patch.items():
            res[bugid] = d = bugs[bugid]
            self.extra_ni[bugid] = data["count"]
            assignee = d["assigned_to"]
            nickname = d["nickname"]

            if not assignee:
                assignee, nickname = nicknames[bugid]

            if not assignee:
                continue

            self.add_auto_ni(bugid, {"mail": assignee, "nickname": nickname})

            common = all_reviewers & data["reviewers_phid"]
            if common:
                reviewer = random.choice(list(common))
                self.add_auto_ni(
                    bugid, {"mail": bz_reviewers[reviewer], "nickname": None}
                )

        return res


if __name__ == "__main__":
    NotLanded().run()

1	# This Source Code Form is subject to the terms of the Mozilla Public
2	# License, v. 2.0. If a copy of the MPL was not distributed with this file,
3	# You can obtain one at http://mozilla.org/MPL/2.0/.
4
5	import base64	×
6	import random	×
7	import re	×
8
9	from libmozdata import utils as lmdutils	×
10	from libmozdata.bugzilla import Bugzilla, BugzillaUser	×
11	from libmozdata.phabricator import (	×
12	PhabricatorAPI,
13	PhabricatorBzNotFoundException,
14	PhabricatorRevisionNotFoundException,
15	)
16
17	from bugbot import utils	×
18	from bugbot.bzcleaner import BzCleaner	×
19
20	PHAB_URL_PAT = re.compile(r"https://phabricator\.services\.mozilla\.com/D([0-9]+)")	×
21
22
23	class NotLanded(BzCleaner):	×
24	def __init__(self):	×
25	super(NotLanded, self).__init__()	×
26	self.nweeks = utils.get_config(self.name(), "number_of_weeks", 2)	×
27	self.nyears = utils.get_config(self.name(), "number_of_years", 2)	×
28	self.phab = PhabricatorAPI(utils.get_login_info()["phab_api_key"])	×
29	self.extra_ni = {}	×
30
31	def description(self):	×
32	return "Open bugs with no activity for {} weeks and a r+ patch which hasn't landed".format(	×
33	self.nweeks
34	)
35
36	def has_assignee(self):	×
37	return True	×
38
39	def get_extra_for_template(self):	×
40	return {"nweeks": self.nweeks}	×
41
42	def get_extra_for_needinfo_template(self):	×
43	self.extra_ni.update(self.get_extra_for_template())	×
44	return self.extra_ni	×
45
46	def columns(self):	×
47	return ["id", "summary", "assignee"]	×
48
49	def handle_bug(self, bug, data):	×
50	if self.has_bot_set_ni(bug):	×
51	return None	×
52
53	bugid = str(bug["id"])	×
54	assignee = bug.get("assigned_to", "")	×
55	if utils.is_no_assignee(assignee):	×
56	assignee = ""	×
57	nickname = ""	×
58	else:
59	nickname = bug["assigned_to_detail"]["nick"]	×
60
61	data[bugid] = {	×
62	"assigned_to": assignee,
63	"nickname": nickname,
64	"deps": set(bug["depends_on"]),
65	}
66
67	return bug	×
68
69	def update_revision_status(self, rev_id, status_type, value):	×
70	try:	×
71	self.phab.request(	×
72	"differential.revision.edit",
73	objectIdentifier=rev_id,
74	transactions=[{"type": status_type, "value": value}],
75	)
76	print(	×
77	f"Successfully updated revision {rev_id} with {status_type} to {value}"
78	)
79	except Exception as e:	×
80	print(	×
81	f"Failed to update revision {rev_id} with {status_type} to {value}: {e}"
82	)
83
84	def filter_bugs(self, bugs):	×
85	# We must remove bugs which have open dependencies (except meta bugs)
86	# because devs may wait for those bugs to be fixed before their patch
87	# can land.
88
89	all_deps = set(dep for info in bugs.values() for dep in info["deps"])	×
90
91	def bug_handler(bug, data):	×
92	if (	×
93	bug["status"] in {"RESOLVED", "VERIFIED", "CLOSED"}
94	or "meta" in bug["keywords"]
95	):
96	data.add(bug["id"])	×
97
98	useless = set()	×
99	Bugzilla(	×
100	bugids=list(all_deps),
101	include_fields=["id", "keywords", "status"],
102	bughandler=bug_handler,
103	bugdata=useless,
104	).get_data().wait()
105
106	for bugid, info in bugs.items():	×
107	# finally deps will contain open bugs which are not meta
108	info["deps"] -= useless	×
109
110	# keep bugs with no deps
111	bugs = {bugid: info for bugid, info in bugs.items() if not info["deps"]}	×
112
113	return bugs	×
114
115	def check_phab(self, attachment, reviewers_phid):	×
116	"""Check if the patch in Phabricator has been r+"""
117	if attachment["is_obsolete"] == 1:	×
118	return None	×
119
120	phab_url = base64.b64decode(attachment["data"]).decode("utf-8")	×
121
122	# extract the revision
123	rev = PHAB_URL_PAT.search(phab_url).group(1)	×
124	try:	×
125	data = self.phab.load_revision(	×
126	rev_id=int(rev), queryKey="all", attachments={"reviewers": 1}
127	)
128	except PhabricatorRevisionNotFoundException:	×
129	return None	×
130
131	# this is a timestamp
132	last_modified = data["fields"]["dateModified"]	×
133	last_modified = lmdutils.get_date_from_timestamp(last_modified)	×
134	if (self.date - last_modified).days <= self.nweeks * 7:	×
135	# Do not do anything if recent changes in the bug
136	return False	×
137
138	reviewers = data["attachments"]["reviewers"]["reviewers"]	×
139
140	if not reviewers:	×
141	return False	×
142
143	for reviewer in reviewers:	×
144	if reviewer["status"] != "accepted":	×
145	return False	×
146	reviewers_phid.add(reviewer["reviewerPHID"])	×
147
148	value = data["fields"]["status"].get("value", "")	×
149	if value == "changes-planned":	×
150	# even if the patch is r+ and not published, some changes may be required
151	# so with the value 'changes-planned', the dev can say it's still a wip
152	return False	×
153
154	if value != "published":	×
155	return True	×
156
157	return False	×
158
159	def handle_attachment(self, attachment, res):	×
160	ct = attachment["content_type"]	×
161	c = None	×
162	if ct == "text/x-phabricator-request":	×
163	if "phab" not in res or res["phab"]:	×
164	c = self.check_phab(attachment, res["reviewers_phid"])	×
165	if c is not None:	×
166	res["phab"] = c	×
167
168	if c is not None:	×
169	attacher = attachment["creator"]	×
170	if "author" in res:	×
171	if attacher in res["author"]:	×
172	res["author"][attacher] += 1	×
173	else:
174	res["author"][attacher] = 1	×
175	else:
176	res["author"] = {attacher: 1}	×
177
178	if "count" in res:	×
179	res["count"] += 1	×
180	else:
181	res["count"] = 1	×
182
183	def get_patch_data(self, bugs):	×
184	"""Get patch information in bugs"""
185
UNCOV 186	def comment_handler(bug, bugid, data):	×
187	# if a comment contains a backout: don't nag
188	for comment in bug["comments"]:	×
189	comment = comment["text"].lower()	×
NEW 190	if "backed out" in comment or "backout" in comment:	×
UNCOV 191	data[bugid]["backout"] = True	×
192
193	def attachment_id_handler(attachments, bugid, data):	×
194	for a in attachments:	×
195	if (	×
196	a["content_type"] == "text/x-phabricator-request"
197	and a["is_obsolete"] == 0
198	):
199	data.append(a["id"])	×
200
201	def attachment_handler(attachments, data):	×
202	for attachment in attachments:	×
203	bugid = str(attachment["bug_id"])	×
204	if bugid in data:	×
205	data[bugid].append(attachment)	×
206	else:
207	data[bugid] = [attachment]	×
208
209	bugids = list(bugs.keys())	×
210	data = {	×
211	bugid: {"backout": False, "author": None, "count": 0} for bugid in bugids
212	}
213
214	# Get the ids of the attachments of interest
215	# to avoid to download images, videos, ...
216	attachment_ids = []	×
217	Bugzilla(	×
218	bugids=bugids,
219	attachmenthandler=attachment_id_handler,
220	attachmentdata=attachment_ids,
221	attachment_include_fields=["is_obsolete", "content_type", "id"],
222	).get_data().wait()
223
224	# Once we've the ids we can get the data
225	attachments_by_bug = {}	×
226	Bugzilla(	×
227	attachmentids=attachment_ids,
228	attachmenthandler=attachment_handler,
229	attachmentdata=attachments_by_bug,
230	attachment_include_fields=[
231	"bug_id",
232	"data",
233	"is_obsolete",
234	"content_type",
235	"id",
236	"creator",
237	],
238	).get_data().wait()
239
240	for bugid, attachments in attachments_by_bug.items():	×
241	res = {"reviewers_phid": set()}	×
242	for attachment in attachments:	×
243	self.handle_attachment(attachment, res)	×
244
245	if "phab" in res:	×
246	if res["phab"]:	×
247	data[bugid]["reviewers_phid"] = res["reviewers_phid"]	×
248	data[bugid]["author"] = res["author"]	×
249	data[bugid]["count"] = res["count"]	×
250
251	data = {bugid: v for bugid, v in data.items() if v["author"]}	×
252
253	if not data:	×
254	return data	×
255
256	Bugzilla(	×
257	bugids=list(data.keys()),
258	commenthandler=comment_handler,
259	commentdata=data,
260	comment_include_fields=["text"],
261	).get_data().wait()
262
NEW 263	for bugid, v in data.items():	×
NEW 264	if data[bugid]["backout"]:	×
NEW 265	phab_url = base64.b64decode(attachment["data"]).decode("utf-8")	×
NEW 266	rev = PHAB_URL_PAT.search(phab_url).group(1)	×
NEW 267	self.update_revision_status(int(rev), "plan-changes", True)	×
268
269	data = {bugid: v for bugid, v in data.items() if not v["backout"]}	×
270
271	return data	×
272
273	def get_bz_userid(self, phids):	×
274	if not phids:	×
275	return {}	×
276
277	try:	×
278	data = self.phab.load_bz_account(user_phids=list(phids))	×
279	users = {x["phid"]: x["id"] for x in data}	×
280	except PhabricatorBzNotFoundException:	×
281	return {}	×
282
283	def handler(user, data):	×
284	data[str(user["id"])] = user["name"]	×
285
286	data = {}	×
287	BugzillaUser(	×
288	user_names=list(users.values()),
289	include_fields=["id", "name"],
290	user_handler=handler,
291	user_data=data,
292	).wait()
293
294	return {phid: data[id] for phid, id in users.items()}	×
295
296	def get_nicks(self, nicknames):	×
297	def handler(user, data):	×
298	data[user["name"]] = user["nick"]	×
299
300	users = set(nicknames.values())	×
301	data = {}	×
302	if users:	×
303	BugzillaUser(	×
304	user_names=list(users),
305	include_fields=["name", "nick"],
306	user_handler=handler,
307	user_data=data,
308	).wait()
309
310	for bugid, name in nicknames.items():	×
311	nicknames[bugid] = (name, data[name])	×
312
313	return nicknames	×
314
315	def get_bz_params(self, date):	×
316	self.date = lmdutils.get_date_ymd(date)	×
317	fields = ["flags", "depends_on"]	×
318	params = {	×
319	"include_fields": fields,
320	"resolution": "---",
321	"f1": "attachment.ispatch",
322	"n2": 1,
323	"f2": "attachments.isobsolete",
324	"f3": "attachments.mimetype",
325	"o3": "anywordssubstr",
326	"v3": "text/x-phabricator-request",
327	"f4": "creation_ts",
328	"o4": "greaterthan",
329	"v4": f"-{self.nyears}y",
330	"f5": "days_elapsed",
331	"o5": "greaterthaneq",
332	"v5": self.nweeks * 7,
333	"n6": 1,
334	"f6": "longdesc",
335	"o6": "casesubstring",
336	"v6": "which didn't land and no activity in this bug for",
337	"f7": "status_whiteboard",
338	"o7": "notsubstring",
339	"v7": "[reminder-test ",
340	}
341
342	return params	×
343
344	def get_bugs(self, date="today", bug_ids=[]):	×
345	bugs = super(NotLanded, self).get_bugs(date=date, bug_ids=bug_ids)	×
346	bugs = self.filter_bugs(bugs)	×
347	bugs_patch = self.get_patch_data(bugs)	×
348	res = {}	×
349
350	reviewers_phid = set()	×
351	nicknames = {}	×
352	for bugid, data in bugs_patch.items():	×
353	reviewers_phid \|= data["reviewers_phid"]	×
354	assignee = bugs[bugid]["assigned_to"]	×
355	if not assignee:	×
356	assignee = max(data["author"], key=data["author"].get)	×
357	nicknames[bugid] = assignee	×
358
359	bz_reviewers = self.get_bz_userid(reviewers_phid)	×
360	all_reviewers = set(bz_reviewers.keys())	×
361	nicknames = self.get_nicks(nicknames)	×
362
363	for bugid, data in bugs_patch.items():	×
364	res[bugid] = d = bugs[bugid]	×
365	self.extra_ni[bugid] = data["count"]	×
366	assignee = d["assigned_to"]	×
367	nickname = d["nickname"]	×
368
369	if not assignee:	×
370	assignee, nickname = nicknames[bugid]	×
371
372	if not assignee:	×
373	continue	×
374
375	self.add_auto_ni(bugid, {"mail": assignee, "nickname": nickname})	×
376
377	common = all_reviewers & data["reviewers_phid"]	×
378	if common:	×
379	reviewer = random.choice(list(common))	×
380	self.add_auto_ni(	×
381	bugid, {"mail": bz_reviewers[reviewer], "nickname": None}
382	)
383
384	return res	×
385
386
387	if __name__ == "__main__":	×
388	NotLanded().run()	×

mozilla / relman-auto-nag / #5167

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous