#5136

Committed 03 Jul 2024 08:38PM UTC coverage: 21.488% (-0.003%) from 21.491%

Build # #5136

Build Type

push

coveralls-python

Committed by

benjaminmah

Commit Message

Removed old needinfo template

Run Details

716 of 3685 branches covered (19.43%)

0 of 4 new or added lines in 1 file covered. (0.0%)

2 existing lines in 1 file now uncovered.

1932 of 8991 relevant lines covered (21.49%)

0.21 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/bugbot/rules/inactive_revision.py

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.

import re
from datetime import datetime
from typing import Dict, List

from dateutil.relativedelta import relativedelta
from jinja2 import Environment, FileSystemLoader, Template
from libmozdata import utils as lmdutils
from libmozdata.connection import Connection
from libmozdata.phabricator import PhabricatorAPI
from tenacity import retry, stop_after_attempt, wait_exponential

from bugbot import utils
from bugbot.bzcleaner import BzCleaner
from bugbot.history import History
from bugbot.user_activity import PHAB_CHUNK_SIZE, UserActivity, UserStatus

PHAB_FILE_NAME_PAT = re.compile(r"phabricator-D([0-9]+)-url\.txt")
PHAB_TABLE_PAT = re.compile(r"^\|\ \[D([0-9]+)\]\(h", flags=re.M)


class InactiveRevision(BzCleaner):
    """Bugs with patches that are waiting for review from inactive reviewers"""

    def __init__(self, old_patch_months: int = 6):
        """Constructor

        Args:
            old_patch_months: number of months since creation of the patch to be
                considered old. If the bug has an old patch, we will mention
                abandon the patch as an option.
        """
        super(InactiveRevision, self).__init__()
        self.phab = PhabricatorAPI(utils.get_login_info()["phab_api_key"])
        self.user_activity = UserActivity(include_fields=["nick"], phab=self.phab)
        self.ni_author_template = self.load_template(
            self.name() + "_needinfo_author.txt"
        )
        self.ni_reviewer_template = self.load_template(
            self.name() + "_needinfo_reviewer.txt"
        )
        self.old_patch_limit = (
            lmdutils.get_date_ymd("today") - relativedelta(months=old_patch_months)
        ).timestamp()

    def description(self):
        return "Bugs with inactive patch reviewers"

    def columns(self):
        return ["id", "summary", "revisions"]

    def get_bugs(self, date="today", bug_ids=[], chunk_size=None):
        bugs = super().get_bugs(date, bug_ids, chunk_size)

        rev_ids = {rev_id for bug in bugs.values() for rev_id in bug["rev_ids"]}
        revisions = self._get_revisions_with_inactive_action(list(rev_ids))

        for bugid, bug in list(bugs.items()):
            inactive_revs = [
                revisions[rev_id] for rev_id in bug["rev_ids"] if rev_id in revisions
            ]
            if inactive_revs:
                bug["revisions"] = inactive_revs
                self._add_needinfo(bugid, inactive_revs)
            else:
                del bugs[bugid]

        self.query_url = utils.get_bz_search_url({"bug_id": ",".join(bugs.keys())})

        return bugs

    def load_template(self, template_filename: str) -> Template:
        env = Environment(loader=FileSystemLoader("templates"))
        template = env.get_template(template_filename)
        return template

    def _add_needinfo(self, bugid: str, inactive_revs: list) -> None:
        has_old_patch = any(
            revision["created_at"] < self.old_patch_limit for revision in inactive_revs
        )

        for revision in inactive_revs:
            last_action_by, _ = self._find_last_action(revision["rev_id"])
            if last_action_by == "author" and revision["reviewers"]:
                ni_mail = revision["reviewers"][0]["phab_username"]
                summary = (
                    "The last action was by the author, so needinfoing the reviewer."
                )
                template = self.ni_reviewer_template
            elif last_action_by == "reviewer":
                ni_mail = revision["author"]["phab_username"]
                summary = (
                    "The last action was by the reviewer, so needinfoing the author."
                )
                template = self.ni_author_template
            else:
                continue

            comment = template.render(
                revisions=[revision],
                nicknames=revision["author"]["nick"],
                reviewers_status_summary=summary,
                has_old_patch=has_old_patch,
                plural=utils.plural,
                documentation=self.get_documentation(),
            )

            self.autofix_changes[bugid] = {
                "comment": {"body": comment},
                "flags": [
                    {
                        "name": "needinfo",
                        "requestee": ni_mail,
                        "status": "?",
                        "new": "true",
                    }
                ],
            }

    def _find_last_action(self, revision_id):
        details = self._fetch_revisions([revision_id])

        if not details:
            return None, None

        revision = details[0]
        author_phid = revision["fields"]["authorPHID"]
        reviewers = [
            reviewer["reviewerPHID"]
            for reviewer in revision["attachments"]["reviewers"]["reviewers"]
        ]

        transactions = self._fetch_revision_transactions(revision["phid"])

        last_transaction = None
        for transaction in transactions:
            if (
                last_transaction is None
                or transaction["dateCreated"] > last_transaction["dateCreated"]
            ):
                last_transaction = transaction

        if last_transaction:
            last_action_by_phid = last_transaction["authorPHID"]
            if last_action_by_phid == author_phid:
                last_action_by = "author"
            elif last_action_by_phid in reviewers:
                last_action_by = "reviewer"
            else:
                last_action_by = "other"
        else:
            last_action_by = "unknown"

        return last_action_by, last_transaction

    def _get_revisions_with_inactive_action(self, rev_ids: list) -> Dict[int, dict]:
        revisions: List[dict] = []
        for _rev_ids in Connection.chunks(rev_ids, PHAB_CHUNK_SIZE):
            for revision in self._fetch_revisions(_rev_ids):
                if (
                    len(revision["attachments"]["reviewers"]["reviewers"]) == 0
                    or revision["fields"]["status"]["value"] != "needs-review"
                    or revision["fields"]["isDraft"]
                ):
                    continue

                reviewers = [
                    {
                        "phid": reviewer["reviewerPHID"],
                        "is_group": reviewer["reviewerPHID"].startswith("PHID-PROJ"),
                        "is_blocking": reviewer["isBlocking"],
                        "is_accepted": reviewer["status"] == "accepted",
                        "is_resigned": reviewer["status"] == "resigned",
                    }
                    for reviewer in revision["attachments"]["reviewers"]["reviewers"]
                ]

                if any(
                    reviewer["is_group"] or reviewer["is_accepted"]
                    for reviewer in reviewers
                ) and not any(
                    not reviewer["is_accepted"]
                    for reviewer in reviewers
                    if reviewer["is_blocking"]
                ):
                    continue

                revisions.append(
                    {
                        "rev_id": revision["id"],
                        "title": revision["fields"]["title"],
                        "created_at": revision["fields"]["dateCreated"],
                        "author_phid": revision["fields"]["authorPHID"],
                        "reviewers": reviewers,
                    }
                )

        user_phids = set()
        for revision in revisions:
            user_phids.add(revision["author_phid"])
            for reviewer in revision["reviewers"]:
                user_phids.add(reviewer["phid"])

        users = self.user_activity.get_phab_users_with_status(
            list(user_phids), keep_active=True
        )

        result: Dict[int, dict] = {}
        for revision in revisions:
            author_info = users[revision["author_phid"]]
            if author_info["status"] != UserStatus.ACTIVE:
                continue

            revision["author"] = author_info

            inactive_reviewers = []
            for reviewer in revision["reviewers"]:
                if reviewer["is_group"]:
                    continue

                reviewer_info = users[reviewer["phid"]]
                if (
                    not reviewer["is_resigned"]
                    and reviewer_info["status"] == UserStatus.ACTIVE
                ):
                    continue

                reviewer["info"] = reviewer_info
                inactive_reviewers.append(reviewer)

            last_action_by, last_transaction = self._find_last_action(
                revision["rev_id"]
            )

            if last_action_by in ["author", "reviewer"]:
                revision["last_action_by"] = last_action_by
                revision["last_transaction"] = last_transaction
                revision["reviewers"] = [
                    {
                        "phab_username": reviewer["info"]["phab_username"],
                        "status_note": self._get_reviewer_status_note(reviewer),
                    }
                    for reviewer in inactive_reviewers
                ]
                result[revision["rev_id"]] = revision

        return result

    @staticmethod
    def _get_reviewer_status_note(reviewer: dict) -> str:
        if reviewer["is_resigned"]:
            return "Resigned from review"

        status = reviewer["info"]["status"]
        if status == UserStatus.UNAVAILABLE:
            until = reviewer["info"]["unavailable_until"]
            if until:
                return_date = datetime.fromtimestamp(until).strftime("%b %-d, %Y")
                return f"Back {return_date}"

            return "Unavailable"

        if status == UserStatus.DISABLED:
            return "Disabled"

        return "Inactive"

    @retry(
        wait=wait_exponential(min=4),
        stop=stop_after_attempt(5),
    )
    def _fetch_revisions(self, ids: list):
        return self.phab.request(
            "differential.revision.search",
            constraints={"ids": ids},
            attachments={"reviewers": True},
        )["data"]

    def _fetch_revision_transactions(self, revision_phid):
        response = self.phab.request(
            "transaction.search", objectIdentifier=revision_phid
        )
        return response["data"]

    def handle_bug(self, bug, data):
        rev_ids = [
            # To avoid loading the attachment content (which can be very large),
            # we extract the revision id from the file name, which is in the
            # format of "phabricator-D{revision_id}-url.txt".
            # len("phabricator-D") == 13
            # len("-url.txt") == 8
            int(attachment["file_name"][13:-8])
            for attachment in bug["attachments"]
            if attachment["content_type"] == "text/x-phabricator-request"
            and PHAB_FILE_NAME_PAT.match(attachment["file_name"])
            and not attachment["is_obsolete"]
        ]

        if not rev_ids:
            return

        # We should not comment about the same patch more than once.
        rev_ids_with_ni = set()
        for comment in bug["comments"]:
            if comment["creator"] == History.BOT and comment["raw_text"].startswith(
                "The following patch"
            ):
                rev_ids_with_ni.update(
                    int(id) for id in PHAB_TABLE_PAT.findall(comment["raw_text"])
                )

        if rev_ids_with_ni:
            rev_ids = [id for id in rev_ids if id not in rev_ids_with_ni]

        if not rev_ids:
            return

        # It will be nicer to show a sorted list of patches
        rev_ids.sort()

        bugid = str(bug["id"])
        data[bugid] = {
            "rev_ids": rev_ids,
        }
        return bug

    def get_bz_params(self, date):
        fields = [
            "comments.raw_text",
            "comments.creator",
            "attachments.file_name",
            "attachments.content_type",
            "attachments.is_obsolete",
        ]
        params = {
            "include_fields": fields,
            "resolution": "---",
            "f1": "attachments.mimetype",
            "o1": "equals",
            "v1": "text/x-phabricator-request",
        }

        return params


if __name__ == "__main__":
    InactiveRevision().run()

1	# This Source Code Form is subject to the terms of the Mozilla Public
2	# License, v. 2.0. If a copy of the MPL was not distributed with this file,
3	# You can obtain one at http://mozilla.org/MPL/2.0/.
4
5	import re	×
6	from datetime import datetime	×
7	from typing import Dict, List	×
8
9	from dateutil.relativedelta import relativedelta	×
10	from jinja2 import Environment, FileSystemLoader, Template	×
11	from libmozdata import utils as lmdutils	×
12	from libmozdata.connection import Connection	×
13	from libmozdata.phabricator import PhabricatorAPI	×
14	from tenacity import retry, stop_after_attempt, wait_exponential	×
15
16	from bugbot import utils	×
17	from bugbot.bzcleaner import BzCleaner	×
18	from bugbot.history import History	×
19	from bugbot.user_activity import PHAB_CHUNK_SIZE, UserActivity, UserStatus	×
20
21	PHAB_FILE_NAME_PAT = re.compile(r"phabricator-D([0-9]+)-url\.txt")	×
22	PHAB_TABLE_PAT = re.compile(r"^\\|\ \[D([0-9]+)\]\(h", flags=re.M)	×
23
24
25	class InactiveRevision(BzCleaner):	×
26	"""Bugs with patches that are waiting for review from inactive reviewers"""
27
28	def __init__(self, old_patch_months: int = 6):	×
29	"""Constructor
30
31	Args:
32	old_patch_months: number of months since creation of the patch to be
33	considered old. If the bug has an old patch, we will mention
34	abandon the patch as an option.
35	"""
36	super(InactiveRevision, self).__init__()	×
37	self.phab = PhabricatorAPI(utils.get_login_info()["phab_api_key"])	×
38	self.user_activity = UserActivity(include_fields=["nick"], phab=self.phab)	×
NEW 39	self.ni_author_template = self.load_template(	×
40	self.name() + "_needinfo_author.txt"
41	)
NEW 42	self.ni_reviewer_template = self.load_template(	×
43	self.name() + "_needinfo_reviewer.txt"
44	)
UNCOV 45	self.old_patch_limit = (	×
46	lmdutils.get_date_ymd("today") - relativedelta(months=old_patch_months)
47	).timestamp()
48
49	def description(self):	×
50	return "Bugs with inactive patch reviewers"	×
51
52	def columns(self):	×
53	return ["id", "summary", "revisions"]	×
54
55	def get_bugs(self, date="today", bug_ids=[], chunk_size=None):	×
56	bugs = super().get_bugs(date, bug_ids, chunk_size)	×
57
58	rev_ids = {rev_id for bug in bugs.values() for rev_id in bug["rev_ids"]}	×
59	revisions = self._get_revisions_with_inactive_action(list(rev_ids))	×
60
61	for bugid, bug in list(bugs.items()):	×
62	inactive_revs = [	×
63	revisions[rev_id] for rev_id in bug["rev_ids"] if rev_id in revisions
64	]
65	if inactive_revs:	×
66	bug["revisions"] = inactive_revs	×
67	self._add_needinfo(bugid, inactive_revs)	×
68	else:
69	del bugs[bugid]	×
70
71	self.query_url = utils.get_bz_search_url({"bug_id": ",".join(bugs.keys())})	×
72
73	return bugs	×
74
75	def load_template(self, template_filename: str) -> Template:	×
UNCOV 76	env = Environment(loader=FileSystemLoader("templates"))	×
77	template = env.get_template(template_filename)	×
78	return template	×
79
80	def _add_needinfo(self, bugid: str, inactive_revs: list) -> None:	×
81	has_old_patch = any(	×
82	revision["created_at"] < self.old_patch_limit for revision in inactive_revs
83	)
84
85	for revision in inactive_revs:	×
86	last_action_by, _ = self._find_last_action(revision["rev_id"])	×
87	if last_action_by == "author" and revision["reviewers"]:	×
88	ni_mail = revision["reviewers"][0]["phab_username"]	×
89	summary = (	×
90	"The last action was by the author, so needinfoing the reviewer."
91	)
NEW 92	template = self.ni_reviewer_template	×
93	elif last_action_by == "reviewer":	×
94	ni_mail = revision["author"]["phab_username"]	×
95	summary = (	×
96	"The last action was by the reviewer, so needinfoing the author."
97	)
NEW 98	template = self.ni_author_template	×
99	else:
100	continue	×
101
102	comment = template.render(	×
103	revisions=[revision],
104	nicknames=revision["author"]["nick"],
105	reviewers_status_summary=summary,
106	has_old_patch=has_old_patch,
107	plural=utils.plural,
108	documentation=self.get_documentation(),
109	)
110
111	self.autofix_changes[bugid] = {	×
112	"comment": {"body": comment},
113	"flags": [
114	{
115	"name": "needinfo",
116	"requestee": ni_mail,
117	"status": "?",
118	"new": "true",
119	}
120	],
121	}
122
123	def _find_last_action(self, revision_id):	×
124	details = self._fetch_revisions([revision_id])	×
125
126	if not details:	×
127	return None, None	×
128
129	revision = details[0]	×
130	author_phid = revision["fields"]["authorPHID"]	×
131	reviewers = [	×
132	reviewer["reviewerPHID"]
133	for reviewer in revision["attachments"]["reviewers"]["reviewers"]
134	]
135
136	transactions = self._fetch_revision_transactions(revision["phid"])	×
137
138	last_transaction = None	×
139	for transaction in transactions:	×
140	if (	×
141	last_transaction is None
142	or transaction["dateCreated"] > last_transaction["dateCreated"]
143	):
144	last_transaction = transaction	×
145
146	if last_transaction:	×
147	last_action_by_phid = last_transaction["authorPHID"]	×
148	if last_action_by_phid == author_phid:	×
149	last_action_by = "author"	×
150	elif last_action_by_phid in reviewers:	×
151	last_action_by = "reviewer"	×
152	else:
153	last_action_by = "other"	×
154	else:
155	last_action_by = "unknown"	×
156
157	return last_action_by, last_transaction	×
158
159	def _get_revisions_with_inactive_action(self, rev_ids: list) -> Dict[int, dict]:	×
160	revisions: List[dict] = []	×
161	for _rev_ids in Connection.chunks(rev_ids, PHAB_CHUNK_SIZE):	×
162	for revision in self._fetch_revisions(_rev_ids):	×
163	if (	×
164	len(revision["attachments"]["reviewers"]["reviewers"]) == 0
165	or revision["fields"]["status"]["value"] != "needs-review"
166	or revision["fields"]["isDraft"]
167	):
168	continue	×
169
170	reviewers = [	×
171	{
172	"phid": reviewer["reviewerPHID"],
173	"is_group": reviewer["reviewerPHID"].startswith("PHID-PROJ"),
174	"is_blocking": reviewer["isBlocking"],
175	"is_accepted": reviewer["status"] == "accepted",
176	"is_resigned": reviewer["status"] == "resigned",
177	}
178	for reviewer in revision["attachments"]["reviewers"]["reviewers"]
179	]
180
181	if any(	×
182	reviewer["is_group"] or reviewer["is_accepted"]
183	for reviewer in reviewers
184	) and not any(
185	not reviewer["is_accepted"]
186	for reviewer in reviewers
187	if reviewer["is_blocking"]
188	):
189	continue	×
190
191	revisions.append(	×
192	{
193	"rev_id": revision["id"],
194	"title": revision["fields"]["title"],
195	"created_at": revision["fields"]["dateCreated"],
196	"author_phid": revision["fields"]["authorPHID"],
197	"reviewers": reviewers,
198	}
199	)
200
201	user_phids = set()	×
202	for revision in revisions:	×
203	user_phids.add(revision["author_phid"])	×
204	for reviewer in revision["reviewers"]:	×
205	user_phids.add(reviewer["phid"])	×
206
207	users = self.user_activity.get_phab_users_with_status(	×
208	list(user_phids), keep_active=True
209	)
210
211	result: Dict[int, dict] = {}	×
212	for revision in revisions:	×
213	author_info = users[revision["author_phid"]]	×
214	if author_info["status"] != UserStatus.ACTIVE:	×
215	continue	×
216
217	revision["author"] = author_info	×
218
219	inactive_reviewers = []	×
220	for reviewer in revision["reviewers"]:	×
221	if reviewer["is_group"]:	×
222	continue	×
223
224	reviewer_info = users[reviewer["phid"]]	×
225	if (	×
226	not reviewer["is_resigned"]
227	and reviewer_info["status"] == UserStatus.ACTIVE
228	):
229	continue	×
230
231	reviewer["info"] = reviewer_info	×
232	inactive_reviewers.append(reviewer)	×
233
234	last_action_by, last_transaction = self._find_last_action(	×
235	revision["rev_id"]
236	)
237
238	if last_action_by in ["author", "reviewer"]:	×
239	revision["last_action_by"] = last_action_by	×
240	revision["last_transaction"] = last_transaction	×
241	revision["reviewers"] = [	×
242	{
243	"phab_username": reviewer["info"]["phab_username"],
244	"status_note": self._get_reviewer_status_note(reviewer),
245	}
246	for reviewer in inactive_reviewers
247	]
248	result[revision["rev_id"]] = revision	×
249
250	return result	×
251
252	@staticmethod	×
253	def _get_reviewer_status_note(reviewer: dict) -> str:	×
254	if reviewer["is_resigned"]:	×
255	return "Resigned from review"	×
256
257	status = reviewer["info"]["status"]	×
258	if status == UserStatus.UNAVAILABLE:	×
259	until = reviewer["info"]["unavailable_until"]	×
260	if until:	×
261	return_date = datetime.fromtimestamp(until).strftime("%b %-d, %Y")	×
262	return f"Back {return_date}"	×
263
264	return "Unavailable"	×
265
266	if status == UserStatus.DISABLED:	×
267	return "Disabled"	×
268
269	return "Inactive"	×
270
271	@retry(	×
272	wait=wait_exponential(min=4),
273	stop=stop_after_attempt(5),
274	)
275	def _fetch_revisions(self, ids: list):	×
276	return self.phab.request(	×
277	"differential.revision.search",
278	constraints={"ids": ids},
279	attachments={"reviewers": True},
280	)["data"]
281
282	def _fetch_revision_transactions(self, revision_phid):	×
283	response = self.phab.request(	×
284	"transaction.search", objectIdentifier=revision_phid
285	)
286	return response["data"]	×
287
288	def handle_bug(self, bug, data):	×
289	rev_ids = [	×
290	# To avoid loading the attachment content (which can be very large),
291	# we extract the revision id from the file name, which is in the
292	# format of "phabricator-D{revision_id}-url.txt".
293	# len("phabricator-D") == 13
294	# len("-url.txt") == 8
295	int(attachment["file_name"][13:-8])
296	for attachment in bug["attachments"]
297	if attachment["content_type"] == "text/x-phabricator-request"
298	and PHAB_FILE_NAME_PAT.match(attachment["file_name"])
299	and not attachment["is_obsolete"]
300	]
301
302	if not rev_ids:	×
303	return	×
304
305	# We should not comment about the same patch more than once.
306	rev_ids_with_ni = set()	×
307	for comment in bug["comments"]:	×
308	if comment["creator"] == History.BOT and comment["raw_text"].startswith(	×
309	"The following patch"
310	):
311	rev_ids_with_ni.update(	×
312	int(id) for id in PHAB_TABLE_PAT.findall(comment["raw_text"])
313	)
314
315	if rev_ids_with_ni:	×
316	rev_ids = [id for id in rev_ids if id not in rev_ids_with_ni]	×
317
318	if not rev_ids:	×
319	return	×
320
321	# It will be nicer to show a sorted list of patches
322	rev_ids.sort()	×
323
324	bugid = str(bug["id"])	×
325	data[bugid] = {	×
326	"rev_ids": rev_ids,
327	}
328	return bug	×
329
330	def get_bz_params(self, date):	×
331	fields = [	×
332	"comments.raw_text",
333	"comments.creator",
334	"attachments.file_name",
335	"attachments.content_type",
336	"attachments.is_obsolete",
337	]
338	params = {	×
339	"include_fields": fields,
340	"resolution": "---",
341	"f1": "attachments.mimetype",
342	"o1": "equals",
343	"v1": "text/x-phabricator-request",
344	}
345
346	return params	×
347
348
349	if __name__ == "__main__":	×
350	InactiveRevision().run()	×

mozilla / relman-auto-nag / #5136

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous