#4349

pending completion

Build # #4349

Build Type

push

coveralls-python

Committed by

sosa-e

Commit Message

Revert "Minimizing config file"

This reverts commit 614159597.

Run Details

564 of 3081 branches covered (18.31%)

24 of 24 new or added lines in 24 files covered. (100.0%)

1804 of 7980 relevant lines covered (22.61%)

0.23 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

63.18

/auto_nag/round_robin.py

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.

import csv
from typing import Dict, Iterator, Set

import requests
from dateutil.relativedelta import relativedelta
from libmozdata import utils as lmdutils
from libmozdata.bugzilla import BugzillaUser
from tenacity import (
    retry,
    retry_if_exception_message,
    stop_after_attempt,
    wait_exponential,
)

from auto_nag import logger, utils
from auto_nag.components import Components
from auto_nag.people import People
from auto_nag.round_robin_calendar import BadFallback, Calendar, InvalidCalendar


class RoundRobin(object):

    _instances: dict = {}

    def __init__(self, rotation_definitions=None, people=None, teams=None):
        self.people = People.get_instance() if people is None else people
        self.components_by_triager: Dict[str, list] = {}
        self.rotation_definitions = (
            RotationDefinitions()
            if rotation_definitions is None
            else rotation_definitions
        )
        self.feed(None if teams is None else set(teams))
        self.nicks = {}
        self.erroneous_bzmail = {}
        utils.init_random()

    @staticmethod
    def get_instance(teams=None):
        if teams is None:
            if None not in RoundRobin._instances:
                RoundRobin._instances[None] = RoundRobin()
            return RoundRobin._instances[None]

        teams = tuple(sorted(teams))
        if teams not in RoundRobin._instances:
            RoundRobin._instances[teams] = RoundRobin(teams=teams)
        return RoundRobin._instances[teams]

    def feed(self, teams: Set[str] = None) -> None:
        """Fetch the rotations calendars.

        Args:
            teams: if provided, only calendars for the specified teams will be
                fetched.
        """

        self.data = {}
        cache = {}

        team_calendars = self.rotation_definitions.fetch_by_teams()
        for team_name, components in team_calendars.items():
            if teams is not None and team_name not in teams:
                continue
            try:
                for component_name, calendar_info in components.items():
                    url = calendar_info["url"]
                    if url not in cache:
                        calendar = cache[url] = Calendar.get(
                            url,
                            calendar_info["fallback"],
                            team_name,
                            people=self.people,
                        )
                    else:
                        calendar = cache[url]
                        if calendar.get_fallback() != calendar_info["fallback"]:
                            raise BadFallback(
                                "Cannot have different fallback triagers for the same calendar"
                            )

                    self.data[component_name] = calendar

            except (BadFallback, InvalidCalendar) as err:
                logger.error(err)
                # If one the team's calendars failed, it is better to fail loud,
                # and disable all team's calendars.
                for component_name in components:
                    if component_name in self.data:
                        del self.data[component_name]

    def get_components(self):
        return list(self.data.keys())

    def get_components_for_triager(self, triager):
        return self.components_by_triager[triager]

    def add_component_for_triager(self, component, triagers):
        if not isinstance(triagers, list):
            triagers = [triagers]
        for triager in triagers:
            if triager in self.components_by_triager:
                self.components_by_triager[triager].add(component)
            else:
                self.components_by_triager[triager] = {component}

    def get_fallback(self, bug):
        pc = bug["product"] + "::" + bug["component"]
        if pc not in self.data:
            mail = bug.get("triage_owner")
        else:
            cal = self.data[pc]
            mail = cal.get_fallback_bzmail()

        return self.people.get_moz_mail(mail)

    def get_erroneous_bzmail(self):
        return self.erroneous_bzmail

    def add_erroneous_bzmail(self, bzmail, prod_comp, cal):
        logger.error(f"No nick for {bzmail} for {prod_comp}")
        fb = cal.get_fallback_mozmail()
        if fb not in self.erroneous_bzmail:
            self.erroneous_bzmail[fb] = {bzmail}
        else:
            self.erroneous_bzmail[fb].add(bzmail)

    def get_nick(self, bzmail, prod_comp, cal):
        if bzmail not in self.nicks:

            def handler(user):
                self.nicks[bzmail] = user["nick"]

            BugzillaUser(user_names=[bzmail], user_handler=handler).wait()

        if bzmail not in self.nicks:
            self.add_erroneous_bzmail(bzmail, prod_comp, cal)
            return None

        return self.nicks[bzmail]

    def get(self, bug, date, only_one=True, has_nick=True):
        pc = bug["product"] + "::" + bug["component"]
        if pc not in self.data:
            mail = bug.get("triage_owner")
            nick = bug.get("triage_owner_detail", {}).get("nick")
            if utils.is_no_assignee(mail):
                mail, nick = None, None

            if mail is None:
                logger.error("No triage owner for {}".format(pc))

            self.add_component_for_triager(pc, mail)

            if has_nick:
                return mail, nick if only_one else [(mail, nick)]
            return mail if only_one else [mail]

        cal = self.data[pc]
        persons = cal.get_persons(date)
        fb = cal.get_fallback_bzmail()
        if not persons or all(p is None for _, p in persons):
            # the fallback is the triage owner
            self.add_component_for_triager(pc, [fb])
            return (fb, self.get_nick(fb, pc, cal)) if has_nick else fb

        bzmails = []
        for _, p in persons:
            bzmails.append(fb if p is None else p)

        self.add_component_for_triager(pc, bzmails)

        if only_one:
            bzmail = bzmails[0]
            if has_nick:
                nick = self.get_nick(bzmail, pc, cal)
                return bzmail, nick
            return bzmail

        if has_nick:
            return [(bzmail, self.get_nick(bzmail, pc, cal)) for bzmail in bzmails]
        return bzmails

    def get_who_to_nag(self, date):
        fallbacks = {}
        date = lmdutils.get_date_ymd(date)
        days = utils.get_config("round-robin", "days_to_nag", 7)
        next_date = date + relativedelta(days=days)
        for cal in set(self.data.values()):
            persons = cal.get_persons(next_date)
            if persons and all(p is not None for _, p in persons):
                continue

            name = cal.get_team_name()
            fb = cal.get_fallback_mozmail()
            if fb not in fallbacks:
                fallbacks[fb] = {}
            if name not in fallbacks[fb]:
                fallbacks[fb][name] = {"nobody": False, "persons": []}
            info = fallbacks[fb][name]

            if not persons:
                info["nobody"] = True
            else:
                people_names = [n for n, p in persons if p is None]
                if people_names:
                    info["persons"] += people_names
        return fallbacks


CalendarDef = Dict[str, str]
ComponentCalendarDefs = Dict[str, CalendarDef]
TeamCalendarDefs = Dict[str, ComponentCalendarDefs]


class RotationDefinitions:
    """Definitions for triage owner rotations"""

    def __init__(self) -> None:
        self.definitions_url = utils.get_private()["round_robin_sheet"]
        self.components = Components.get_instance()

    def fetch_by_teams(self) -> TeamCalendarDefs:
        """Fetch the triage owner rotation definitions and group them by team

        Returns:
            A dictionary that maps each component to its calendar and fallback
            person. The components are grouped by their teams. The following is
            the shape of the returned dictionary:
            {
                team_name: {
                    component_name:{
                            "fallback": "the name of the fallback person",
                            "calendar": "the URL for the rotation calendar"
                    }
                    ...
                }
                ...
            }
        """

        teams: TeamCalendarDefs = {}
        seen = set()
        for row in csv.DictReader(self.get_definitions_csv_lines()):
            team_name = row["Team Name"]
            scope = row["Calendar Scope"]
            fallback_triager = row["Fallback Triager"]
            calendar_url = row["Calendar URL"]

            if (team_name, scope) in seen:
                logger.error(
                    "The triage owner rotation definitions show more than one "
                    "entry for the %s team with the component scope '%s'",
                    team_name,
                    scope,
                )
            else:
                seen.add((team_name, scope))

            if team_name in teams:
                component_calendar = teams[team_name]
            else:
                component_calendar = teams[team_name] = {}

            if scope == "All Team's Components":
                team_components = self.components.get_team_components(team_name)
                components_to_add = [
                    str(component_name)
                    for component_name in team_components
                    if str(component_name) not in component_calendar
                ]
            else:
                components_to_add = [scope]

            for component_name in components_to_add:
                component_calendar[component_name] = {
                    "fallback": fallback_triager,
                    "url": calendar_url,
                }

        return teams

    def get_definitions_csv_lines(self) -> Iterator[str]:
        """Get the definitions for the triage owner rotations in CSV format.

        Returns:
            An iterator where each iteration should return a line from the CSV
            file. The first line will be the headers::
                - Team Name
                - Calendar Scope
                - Fallback Triager
                - Calendar URL"
        """
        return self._fetch_definitions_csv()

    @retry(
        retry=retry_if_exception_message(match=r"^\d{3} Server Error"),
        wait=wait_exponential(min=4),
        stop=stop_after_attempt(3),
    )
    def _fetch_definitions_csv(self) -> Iterator[str]:
        resp = requests.get(self.definitions_url)
        resp.raise_for_status()

        return resp.iter_lines(decode_unicode=True)

1	# This Source Code Form is subject to the terms of the Mozilla Public
2	# License, v. 2.0. If a copy of the MPL was not distributed with this file,
3	# You can obtain one at http://mozilla.org/MPL/2.0/.
4
5	import csv	1✔
6	from typing import Dict, Iterator, Set	1✔
7
8	import requests	1✔
9	from dateutil.relativedelta import relativedelta	1✔
10	from libmozdata import utils as lmdutils	1✔
11	from libmozdata.bugzilla import BugzillaUser	1✔
12	from tenacity import (	1✔
13	retry,
14	retry_if_exception_message,
15	stop_after_attempt,
16	wait_exponential,
17	)
18
19	from auto_nag import logger, utils	1✔
20	from auto_nag.components import Components	1✔
21	from auto_nag.people import People	1✔
22	from auto_nag.round_robin_calendar import BadFallback, Calendar, InvalidCalendar	1✔
23
24
25	class RoundRobin(object):	1✔
26
27	_instances: dict = {}	1✔
28
29	def __init__(self, rotation_definitions=None, people=None, teams=None):	1✔
30	self.people = People.get_instance() if people is None else people	1✔
31	self.components_by_triager: Dict[str, list] = {}	1✔
32	self.rotation_definitions = (	1✔
33	RotationDefinitions()
34	if rotation_definitions is None
35	else rotation_definitions
36	)
37	self.feed(None if teams is None else set(teams))	1✔
38	self.nicks = {}	1✔
39	self.erroneous_bzmail = {}	1✔
40	utils.init_random()	1✔
41
42	@staticmethod	1✔
43	def get_instance(teams=None):	1✔
44	if teams is None:	×
45	if None not in RoundRobin._instances:	×
46	RoundRobin._instances[None] = RoundRobin()	×
47	return RoundRobin._instances[None]	×
48
49	teams = tuple(sorted(teams))	×
50	if teams not in RoundRobin._instances:	×
51	RoundRobin._instances[teams] = RoundRobin(teams=teams)	×
52	return RoundRobin._instances[teams]	×
53
54	def feed(self, teams: Set[str] = None) -> None:	1✔
55	"""Fetch the rotations calendars.
56
57	Args:
58	teams: if provided, only calendars for the specified teams will be
59	fetched.
60	"""
61
62	self.data = {}	1✔
63	cache = {}	1✔
64
65	team_calendars = self.rotation_definitions.fetch_by_teams()	1✔
66	for team_name, components in team_calendars.items():	1✔
67	if teams is not None and team_name not in teams:	1!
68	continue	×
69	try:	1✔
70	for component_name, calendar_info in components.items():	1✔
71	url = calendar_info["url"]	1✔
72	if url not in cache:	1✔
73	calendar = cache[url] = Calendar.get(	1✔
74	url,
75	calendar_info["fallback"],
76	team_name,
77	people=self.people,
78	)
79	else:
80	calendar = cache[url]	1✔
81	if calendar.get_fallback() != calendar_info["fallback"]:	1!
82	raise BadFallback(	×
83	"Cannot have different fallback triagers for the same calendar"
84	)
85
86	self.data[component_name] = calendar	1✔
87
88	except (BadFallback, InvalidCalendar) as err:	×
89	logger.error(err)	×
90	# If one the team's calendars failed, it is better to fail loud,
91	# and disable all team's calendars.
92	for component_name in components:	×
93	if component_name in self.data:	×
94	del self.data[component_name]	×
95
96	def get_components(self):	1✔
97	return list(self.data.keys())	×
98
99	def get_components_for_triager(self, triager):	1✔
100	return self.components_by_triager[triager]	×
101
102	def add_component_for_triager(self, component, triagers):	1✔
103	if not isinstance(triagers, list):	1✔
104	triagers = [triagers]	1✔
105	for triager in triagers:	1✔
106	if triager in self.components_by_triager:	1✔
107	self.components_by_triager[triager].add(component)	1✔
108	else:
109	self.components_by_triager[triager] = {component}	1✔
110
111	def get_fallback(self, bug):	1✔
112	pc = bug["product"] + "::" + bug["component"]	×
113	if pc not in self.data:	×
114	mail = bug.get("triage_owner")	×
115	else:
116	cal = self.data[pc]	×
117	mail = cal.get_fallback_bzmail()	×
118
119	return self.people.get_moz_mail(mail)	×
120
121	def get_erroneous_bzmail(self):	1✔
122	return self.erroneous_bzmail	×
123
124	def add_erroneous_bzmail(self, bzmail, prod_comp, cal):	1✔
125	logger.error(f"No nick for {bzmail} for {prod_comp}")	×
126	fb = cal.get_fallback_mozmail()	×
127	if fb not in self.erroneous_bzmail:	×
128	self.erroneous_bzmail[fb] = {bzmail}	×
129	else:
130	self.erroneous_bzmail[fb].add(bzmail)	×
131
132	def get_nick(self, bzmail, prod_comp, cal):	1✔
133	if bzmail not in self.nicks:	×
134
135	def handler(user):	×
136	self.nicks[bzmail] = user["nick"]	×
137
138	BugzillaUser(user_names=[bzmail], user_handler=handler).wait()	×
139
140	if bzmail not in self.nicks:	×
141	self.add_erroneous_bzmail(bzmail, prod_comp, cal)	×
142	return None	×
143
144	return self.nicks[bzmail]	×
145
146	def get(self, bug, date, only_one=True, has_nick=True):	1✔
147	pc = bug["product"] + "::" + bug["component"]	1✔
148	if pc not in self.data:	1✔
149	mail = bug.get("triage_owner")	1✔
150	nick = bug.get("triage_owner_detail", {}).get("nick")	1✔
151	if utils.is_no_assignee(mail):	1!
152	mail, nick = None, None	×
153
154	if mail is None:	1!
155	logger.error("No triage owner for {}".format(pc))	×
156
157	self.add_component_for_triager(pc, mail)	1✔
158
159	if has_nick:	1!
160	return mail, nick if only_one else [(mail, nick)]	1✔
161	return mail if only_one else [mail]	×
162
163	cal = self.data[pc]	1✔
164	persons = cal.get_persons(date)	1✔
165	fb = cal.get_fallback_bzmail()	1✔
166	if not persons or all(p is None for _, p in persons):	1!
167	# the fallback is the triage owner
168	self.add_component_for_triager(pc, [fb])	1✔
169	return (fb, self.get_nick(fb, pc, cal)) if has_nick else fb	1✔
170
171	bzmails = []	1✔
172	for _, p in persons:	1✔
173	bzmails.append(fb if p is None else p)	1✔
174
175	self.add_component_for_triager(pc, bzmails)	1✔
176
177	if only_one:	1!
178	bzmail = bzmails[0]	1✔
179	if has_nick:	1!
180	nick = self.get_nick(bzmail, pc, cal)	1✔
181	return bzmail, nick	1✔
182	return bzmail	×
183
184	if has_nick:	×
185	return [(bzmail, self.get_nick(bzmail, pc, cal)) for bzmail in bzmails]	×
186	return bzmails	×
187
188	def get_who_to_nag(self, date):	1✔
189	fallbacks = {}	1✔
190	date = lmdutils.get_date_ymd(date)	1✔
191	days = utils.get_config("round-robin", "days_to_nag", 7)	1✔
192	next_date = date + relativedelta(days=days)	1✔
193	for cal in set(self.data.values()):	1✔
194	persons = cal.get_persons(next_date)	1✔
195	if persons and all(p is not None for _, p in persons):	1✔
196	continue	1✔
197
198	name = cal.get_team_name()	1✔
199	fb = cal.get_fallback_mozmail()	1✔
200	if fb not in fallbacks:	1✔
201	fallbacks[fb] = {}	1✔
202	if name not in fallbacks[fb]:	1✔
203	fallbacks[fb][name] = {"nobody": False, "persons": []}	1✔
204	info = fallbacks[fb][name]	1✔
205
206	if not persons:	1!
207	info["nobody"] = True	1✔
208	else:
209	people_names = [n for n, p in persons if p is None]	×
210	if people_names:	×
211	info["persons"] += people_names	×
212	return fallbacks	1✔
213
214
215	CalendarDef = Dict[str, str]	1✔
216	ComponentCalendarDefs = Dict[str, CalendarDef]	1✔
217	TeamCalendarDefs = Dict[str, ComponentCalendarDefs]	1✔
218
219
220	class RotationDefinitions:	1✔
221	"""Definitions for triage owner rotations"""
222
223	def __init__(self) -> None:	1✔
224	self.definitions_url = utils.get_private()["round_robin_sheet"]	×
225	self.components = Components.get_instance()	×
226
227	def fetch_by_teams(self) -> TeamCalendarDefs:	1✔
228	"""Fetch the triage owner rotation definitions and group them by team
229
230	Returns:
231	A dictionary that maps each component to its calendar and fallback
232	person. The components are grouped by their teams. The following is
233	the shape of the returned dictionary:
234	{
235	team_name: {
236	component_name:{
237	"fallback": "the name of the fallback person",
238	"calendar": "the URL for the rotation calendar"
239	}
240	...
241	}
242	...
243	}
244	"""
245
246	teams: TeamCalendarDefs = {}	1✔
247	seen = set()	1✔
248	for row in csv.DictReader(self.get_definitions_csv_lines()):	1✔
249	team_name = row["Team Name"]	1✔
250	scope = row["Calendar Scope"]	1✔
251	fallback_triager = row["Fallback Triager"]	1✔
252	calendar_url = row["Calendar URL"]	1✔
253
254	if (team_name, scope) in seen:	1!
255	logger.error(	×
256	"The triage owner rotation definitions show more than one "
257	"entry for the %s team with the component scope '%s'",
258	team_name,
259	scope,
260	)
261	else:
262	seen.add((team_name, scope))	1✔
263
264	if team_name in teams:	1✔
265	component_calendar = teams[team_name]	1✔
266	else:
267	component_calendar = teams[team_name] = {}	1✔
268
269	if scope == "All Team's Components":	1!
270	team_components = self.components.get_team_components(team_name)	×
271	components_to_add = [	×
272	str(component_name)
273	for component_name in team_components
274	if str(component_name) not in component_calendar
275	]
276	else:
277	components_to_add = [scope]	1✔
278
279	for component_name in components_to_add:	1✔
280	component_calendar[component_name] = {	1✔
281	"fallback": fallback_triager,
282	"url": calendar_url,
283	}
284
285	return teams	1✔
286
287	def get_definitions_csv_lines(self) -> Iterator[str]:	1✔
288	"""Get the definitions for the triage owner rotations in CSV format.
289
290	Returns:
291	An iterator where each iteration should return a line from the CSV
292	file. The first line will be the headers::
293	- Team Name
294	- Calendar Scope
295	- Fallback Triager
296	- Calendar URL"
297	"""
298	return self._fetch_definitions_csv()	×
299
300	@retry(	1✔
301	retry=retry_if_exception_message(match=r"^\d{3} Server Error"),
302	wait=wait_exponential(min=4),
303	stop=stop_after_attempt(3),
304	)
305	def _fetch_definitions_csv(self) -> Iterator[str]:	1✔
306	resp = requests.get(self.definitions_url)	×
307	resp.raise_for_status()	×
308
309	return resp.iter_lines(decode_unicode=True)	×

mozilla / relman-auto-nag / #4349

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous