11327586461

Committed 14 Oct 2024 12:31PM CUT coverage: 35.273%. First build

Build # 11327586461

Build Type

Pull #1845

github

Committed by

web-flow

Commit Message

Merge 0487caa15 into f558f7daa

Pull Request Pull Request #1845: build(deps): bump coverage[toml] from 7.6.1 to 7.6.3

Run Details

994 of 2818 relevant lines covered (35.27%)

1.05 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

16.47

/mozregression/fetch_build_info.py

"""
This module offers an API to get information for one build.

The public API is composed of two classes, :class:`NightlyInfoFetcher` and
:class:`IntegrationInfoFetcher`, able to return
:class:`mozregression.build_info.BuildInfo` instances.
"""

from __future__ import absolute_import

import os
import re
from datetime import datetime
from threading import Lock, Thread

import requests
import taskcluster
from mozlog import get_proxy_logger
from taskcluster.exceptions import TaskclusterFailure

from mozregression.build_info import IntegrationBuildInfo, NightlyBuildInfo
from mozregression.errors import BuildInfoNotFound, MozRegressionError
from mozregression.json_pushes import JsonPushes, Push
from mozregression.network import retry_get, url_links

LOG = get_proxy_logger(__name__)


class InfoFetcher(object):
    def __init__(self, fetch_config):
        self.fetch_config = fetch_config
        self.build_regex = re.compile(fetch_config.build_regex())
        self.build_info_regex = re.compile(fetch_config.build_info_regex())

    def _update_build_info_from_txt(self, build_info):
        LOG.debug("Update build info from {}".format(build_info))
        if "build_txt_url" in build_info:
            build_info.update(self._fetch_txt_info(build_info["build_txt_url"]))

    def _fetch_txt_info(self, url):
        """
        Retrieve information from a build information txt file.

        Returns a dict with keys repository and changeset if information
        is found.
        """
        LOG.debug("Fetching txt info from {}".format(url))
        data = {}
        response = retry_get(url)
        for line in response.text.splitlines():
            if "/rev/" in line:
                repository, changeset = line.split("/rev/")
                data["repository"] = repository
                data["changeset"] = changeset
                break
        if not data:
            # the txt file could be in an old format:
            # DATE CHANGESET
            # we can try to extract that to get the changeset at least.
            matched = re.match(r"^\d+ (\w+)$", response.text.strip())
            if matched:
                data["changeset"] = matched.group(1)
        return data

    def find_build_info(self, changeset_or_date, fetch_txt_info=True):
        """
        Abstract method to retrieve build information over the internet for
        one build.

        This returns a :class:`BuildInfo` instance that contain build
        information.

        Note that this method may raise :class:`BuildInfoNotFound` on error.
        """
        raise NotImplementedError


class IntegrationInfoFetcher(InfoFetcher):
    def __init__(self, fetch_config):
        InfoFetcher.__init__(self, fetch_config)
        self.jpushes = JsonPushes(branch=fetch_config.integration_branch)
        options = fetch_config.tk_options()
        self.index = taskcluster.Index(options)
        self.queue = taskcluster.Queue(options)

    def find_build_info(self, push):
        """
        Find build info for an integration build, given a Push, a changeset or a
        date/datetime.

        if `push` is not an instance of Push (e.g. it is a date, datetime, or
        string representing the changeset), a query to json pushes will be
        done.

        Return a :class:`IntegrationBuildInfo` instance.
        """
        if not isinstance(push, Push):
            try:
                push = self.jpushes.push(push)
            except MozRegressionError as exc:
                raise BuildInfoNotFound(str(exc))

        changeset = push.changeset

        tk_routes = self.fetch_config.tk_routes(push)
        try:
            task_id = None
            stored_failure = None
            for tk_route in tk_routes:
                LOG.debug("using taskcluster route %r" % tk_route)
                try:
                    task_id = self.index.findTask(tk_route)["taskId"]
                except TaskclusterFailure as ex:
                    LOG.debug("nothing found via route %r" % tk_route)
                    stored_failure = ex
                    continue
                if task_id:
                    status = self.queue.status(task_id)["status"]
                    break
            if not task_id:
                raise stored_failure
        except TaskclusterFailure:
            raise BuildInfoNotFound(
                "Unable to find build info using the"
                " taskcluster route %r" % self.fetch_config.tk_route(push)
            )

        # find a completed run for that task
        run_id, build_date = None, None
        for run in reversed(status["runs"]):
            if run["state"] == "completed":
                run_id = run["runId"]
                try:
                    build_date = datetime.strptime(run["resolved"], "%Y-%m-%dT%H:%M:%S.%fZ")
                except ValueError:
                    build_date = datetime.strptime(run["resolved"], "%Y-%m-%dT%H:%M:%S.%f+00:00")
                break

        if run_id is None:
            raise BuildInfoNotFound("Unable to find completed runs for task %s" % task_id)
        artifacts = self.queue.listArtifacts(task_id, run_id)["artifacts"]

        # look over the artifacts of that run
        build_url = None
        for a in artifacts:
            name = os.path.basename(a["name"])
            if self.build_regex.search(name):
                meth = self.queue.buildUrl
                if self.fetch_config.tk_needs_auth():
                    meth = self.queue.buildSignedUrl
                build_url = meth("getArtifact", task_id, run_id, a["name"])
                break
        if build_url is None:
            raise BuildInfoNotFound(
                "unable to find a build url for the" " changeset %r" % changeset
            )

        if self.fetch_config.app_name == "gve":
            # Check taskcluster URL to make sure artifact is still around.
            # build_url is an alias that redirects via a 303 status code.
            status_code = requests.head(build_url, allow_redirects=True).status_code
            if status_code != 200:
                error = f"Taskcluster file {build_url} not available (status code: {status_code})."
                raise BuildInfoNotFound(error)
        return IntegrationBuildInfo(
            self.fetch_config,
            build_url=build_url,
            build_date=build_date,
            changeset=changeset,
            repo_url=self.jpushes.repo_url,
            task_id=task_id,
        )


class NightlyInfoFetcher(InfoFetcher):
    def __init__(self, fetch_config):
        InfoFetcher.__init__(self, fetch_config)
        self._cache_months = {}
        self._lock = Lock()
        self._fetch_lock = Lock()

    def _fetch_build_info_from_url(self, url, index, lst):
        """
        Retrieve information from a build folder url.

        Stores in a list the url index and a dict instance with keys
        build_url and build_txt_url if respectively a build file and a
        build info file are found for the url.
        """
        LOG.debug("Fetching build info from {}".format(url))
        data = {}
        if not url.endswith("/"):
            url += "/"
        links = url_links(url)
        if not self.fetch_config.has_build_info:
            links += url_links(self.fetch_config.get_nightly_info_url(url))
        for link in links:
            name = os.path.basename(link)
            if "build_url" not in data and self.build_regex.match(name):
                data["build_url"] = link
            elif "build_txt_url" not in data and self.build_info_regex.match(name):
                data["build_txt_url"] = link
        if data:
            # Check that we found all required data. The URL in build_url is
            # required. build_txt_url is optional.
            if "build_url" not in data:
                raise BuildInfoNotFound(
                    "Failed to find a build file in directory {} that "
                    "matches regex '{}'".format(url, self.build_regex.pattern)
                )

            with self._fetch_lock:
                lst.append((index, data))

    def _get_month_links(self, url):
        with self._lock:
            if url not in self._cache_months:
                self._cache_months[url] = url_links(url)
            return self._cache_months[url]

    def _get_urls(self, date):
        """
        Get the url list of the build folder for a given date.

        This methods needs to be thread-safe as it is used in
        :meth:`NightlyBuildData.get_build_url`.
        """
        LOG.debug("Get URLs for {}".format(date))
        url = self.fetch_config.get_nightly_base_url(date)
        link_regex = re.compile(self.fetch_config.get_nightly_repo_regex(date))

        month_links = self._get_month_links(url)

        # first parse monthly list to get correct directory
        matches = []
        for dirlink in month_links:
            if link_regex.search(dirlink):
                matches.append(dirlink)
        # the most recent build urls first
        matches.reverse()
        return matches

    def find_build_info(self, date, fetch_txt_info=True, max_workers=2):
        """
        Find build info for a nightly build, given a date.

        Returns a :class:`NightlyBuildInfo` instance.
        """
        # getting a valid build for a given date on nightly is tricky.
        # there is multiple possible builds folders for one date,
        # and some of them may be invalid (without binary for example)

        # to save time, we will try multiple build folders at the same
        # time in some threads. The first good one found is returned.
        try:
            build_urls = self._get_urls(date)
            LOG.debug("got build_urls %s" % build_urls)
        except requests.HTTPError as exc:
            raise BuildInfoNotFound(str(exc))
        build_info = None

        valid_builds = []
        while build_urls:
            some = build_urls[:max_workers]
            threads = [
                Thread(target=self._fetch_build_info_from_url, args=(url, i, valid_builds))
                for i, url in enumerate(some)
            ]
            for thread in threads:
                thread.daemon = True
                thread.start()
            for thread in threads:
                while thread.is_alive():
                    thread.join(0.1)
            LOG.debug("got valid_builds %s" % valid_builds)
            if valid_builds:
                infos = sorted(valid_builds, key=lambda b: b[0])[0][1]
                if fetch_txt_info:
                    self._update_build_info_from_txt(infos)

                build_info = NightlyBuildInfo(
                    self.fetch_config,
                    build_url=infos["build_url"],
                    build_date=date,
                    changeset=infos.get("changeset"),
                    repo_url=infos.get("repository"),
                )
                break
            build_urls = build_urls[max_workers:]

        if build_info is None:
            raise BuildInfoNotFound("Unable to find build info for %s" % date)

        return build_info

1	"""
2	This module offers an API to get information for one build.
3
4	The public API is composed of two classes, :class:`NightlyInfoFetcher` and
5	:class:`IntegrationInfoFetcher`, able to return
6	:class:`mozregression.build_info.BuildInfo` instances.
7	"""
8
9	from __future__ import absolute_import	3✔
10
11	import os	3✔
12	import re	3✔
13	from datetime import datetime	3✔
14	from threading import Lock, Thread	3✔
15
16	import requests	3✔
17	import taskcluster	3✔
18	from mozlog import get_proxy_logger	3✔
19	from taskcluster.exceptions import TaskclusterFailure	3✔
20
21	from mozregression.build_info import IntegrationBuildInfo, NightlyBuildInfo	3✔
22	from mozregression.errors import BuildInfoNotFound, MozRegressionError	3✔
23	from mozregression.json_pushes import JsonPushes, Push	3✔
24	from mozregression.network import retry_get, url_links	3✔
25
26	LOG = get_proxy_logger(__name__)	3✔
27
28
29	class InfoFetcher(object):	3✔
30	def __init__(self, fetch_config):	3✔
31	self.fetch_config = fetch_config	×
32	self.build_regex = re.compile(fetch_config.build_regex())	×
33	self.build_info_regex = re.compile(fetch_config.build_info_regex())	×
34
35	def _update_build_info_from_txt(self, build_info):	3✔
36	LOG.debug("Update build info from {}".format(build_info))	×
37	if "build_txt_url" in build_info:	×
38	build_info.update(self._fetch_txt_info(build_info["build_txt_url"]))	×
39
40	def _fetch_txt_info(self, url):	3✔
41	"""
42	Retrieve information from a build information txt file.
43
44	Returns a dict with keys repository and changeset if information
45	is found.
46	"""
47	LOG.debug("Fetching txt info from {}".format(url))	×
48	data = {}	×
49	response = retry_get(url)	×
50	for line in response.text.splitlines():	×
51	if "/rev/" in line:	×
52	repository, changeset = line.split("/rev/")	×
53	data["repository"] = repository	×
54	data["changeset"] = changeset	×
55	break	×
56	if not data:	×
57	# the txt file could be in an old format:
58	# DATE CHANGESET
59	# we can try to extract that to get the changeset at least.
60	matched = re.match(r"^\d+ (\w+)$", response.text.strip())	×
61	if matched:	×
62	data["changeset"] = matched.group(1)	×
63	return data	×
64
65	def find_build_info(self, changeset_or_date, fetch_txt_info=True):	3✔
66	"""
67	Abstract method to retrieve build information over the internet for
68	one build.
69
70	This returns a :class:`BuildInfo` instance that contain build
71	information.
72
73	Note that this method may raise :class:`BuildInfoNotFound` on error.
74	"""
75	raise NotImplementedError
76
77
78	class IntegrationInfoFetcher(InfoFetcher):	3✔
79	def __init__(self, fetch_config):	3✔
80	InfoFetcher.__init__(self, fetch_config)	×
81	self.jpushes = JsonPushes(branch=fetch_config.integration_branch)	×
82	options = fetch_config.tk_options()	×
83	self.index = taskcluster.Index(options)	×
84	self.queue = taskcluster.Queue(options)	×
85
86	def find_build_info(self, push):	3✔
87	"""
88	Find build info for an integration build, given a Push, a changeset or a
89	date/datetime.
90
91	if `push` is not an instance of Push (e.g. it is a date, datetime, or
92	string representing the changeset), a query to json pushes will be
93	done.
94
95	Return a :class:`IntegrationBuildInfo` instance.
96	"""
97	if not isinstance(push, Push):	×
98	try:	×
99	push = self.jpushes.push(push)	×
100	except MozRegressionError as exc:	×
101	raise BuildInfoNotFound(str(exc))	×
102
103	changeset = push.changeset	×
104
105	tk_routes = self.fetch_config.tk_routes(push)	×
106	try:	×
107	task_id = None	×
108	stored_failure = None	×
109	for tk_route in tk_routes:	×
110	LOG.debug("using taskcluster route %r" % tk_route)	×
111	try:	×
112	task_id = self.index.findTask(tk_route)["taskId"]	×
113	except TaskclusterFailure as ex:	×
114	LOG.debug("nothing found via route %r" % tk_route)	×
115	stored_failure = ex	×
116	continue	×
117	if task_id:	×
118	status = self.queue.status(task_id)["status"]	×
119	break	×
120	if not task_id:	×
121	raise stored_failure	×
122	except TaskclusterFailure:	×
123	raise BuildInfoNotFound(	×
124	"Unable to find build info using the"
125	" taskcluster route %r" % self.fetch_config.tk_route(push)
126	)
127
128	# find a completed run for that task
129	run_id, build_date = None, None	×
130	for run in reversed(status["runs"]):	×
131	if run["state"] == "completed":	×
132	run_id = run["runId"]	×
133	try:	×
134	build_date = datetime.strptime(run["resolved"], "%Y-%m-%dT%H:%M:%S.%fZ")	×
135	except ValueError:	×
136	build_date = datetime.strptime(run["resolved"], "%Y-%m-%dT%H:%M:%S.%f+00:00")	×
137	break	×
138
139	if run_id is None:	×
140	raise BuildInfoNotFound("Unable to find completed runs for task %s" % task_id)	×
141	artifacts = self.queue.listArtifacts(task_id, run_id)["artifacts"]	×
142
143	# look over the artifacts of that run
144	build_url = None	×
145	for a in artifacts:	×
146	name = os.path.basename(a["name"])	×
147	if self.build_regex.search(name):	×
148	meth = self.queue.buildUrl	×
149	if self.fetch_config.tk_needs_auth():	×
150	meth = self.queue.buildSignedUrl	×
151	build_url = meth("getArtifact", task_id, run_id, a["name"])	×
152	break	×
153	if build_url is None:	×
154	raise BuildInfoNotFound(	×
155	"unable to find a build url for the" " changeset %r" % changeset
156	)
157
158	if self.fetch_config.app_name == "gve":	×
159	# Check taskcluster URL to make sure artifact is still around.
160	# build_url is an alias that redirects via a 303 status code.
161	status_code = requests.head(build_url, allow_redirects=True).status_code	×
162	if status_code != 200:	×
163	error = f"Taskcluster file {build_url} not available (status code: {status_code})."	×
164	raise BuildInfoNotFound(error)	×
165	return IntegrationBuildInfo(	×
166	self.fetch_config,
167	build_url=build_url,
168	build_date=build_date,
169	changeset=changeset,
170	repo_url=self.jpushes.repo_url,
171	task_id=task_id,
172	)
173
174
175	class NightlyInfoFetcher(InfoFetcher):	3✔
176	def __init__(self, fetch_config):	3✔
177	InfoFetcher.__init__(self, fetch_config)	×
178	self._cache_months = {}	×
179	self._lock = Lock()	×
180	self._fetch_lock = Lock()	×
181
182	def _fetch_build_info_from_url(self, url, index, lst):	3✔
183	"""
184	Retrieve information from a build folder url.
185
186	Stores in a list the url index and a dict instance with keys
187	build_url and build_txt_url if respectively a build file and a
188	build info file are found for the url.
189	"""
190	LOG.debug("Fetching build info from {}".format(url))	×
191	data = {}	×
192	if not url.endswith("/"):	×
193	url += "/"	×
194	links = url_links(url)	×
195	if not self.fetch_config.has_build_info:	×
196	links += url_links(self.fetch_config.get_nightly_info_url(url))	×
197	for link in links:	×
198	name = os.path.basename(link)	×
199	if "build_url" not in data and self.build_regex.match(name):	×
200	data["build_url"] = link	×
201	elif "build_txt_url" not in data and self.build_info_regex.match(name):	×
202	data["build_txt_url"] = link	×
203	if data:	×
204	# Check that we found all required data. The URL in build_url is
205	# required. build_txt_url is optional.
206	if "build_url" not in data:	×
207	raise BuildInfoNotFound(	×
208	"Failed to find a build file in directory {} that "
209	"matches regex '{}'".format(url, self.build_regex.pattern)
210	)
211
212	with self._fetch_lock:	×
213	lst.append((index, data))	×
214
215	def _get_month_links(self, url):	3✔
216	with self._lock:	×
217	if url not in self._cache_months:	×
218	self._cache_months[url] = url_links(url)	×
219	return self._cache_months[url]	×
220
221	def _get_urls(self, date):	3✔
222	"""
223	Get the url list of the build folder for a given date.
224
225	This methods needs to be thread-safe as it is used in
226	:meth:`NightlyBuildData.get_build_url`.
227	"""
228	LOG.debug("Get URLs for {}".format(date))	×
229	url = self.fetch_config.get_nightly_base_url(date)	×
230	link_regex = re.compile(self.fetch_config.get_nightly_repo_regex(date))	×
231
232	month_links = self._get_month_links(url)	×
233
234	# first parse monthly list to get correct directory
235	matches = []	×
236	for dirlink in month_links:	×
237	if link_regex.search(dirlink):	×
238	matches.append(dirlink)	×
239	# the most recent build urls first
240	matches.reverse()	×
241	return matches	×
242
243	def find_build_info(self, date, fetch_txt_info=True, max_workers=2):	3✔
244	"""
245	Find build info for a nightly build, given a date.
246
247	Returns a :class:`NightlyBuildInfo` instance.
248	"""
249	# getting a valid build for a given date on nightly is tricky.
250	# there is multiple possible builds folders for one date,
251	# and some of them may be invalid (without binary for example)
252
253	# to save time, we will try multiple build folders at the same
254	# time in some threads. The first good one found is returned.
255	try:	×
256	build_urls = self._get_urls(date)	×
257	LOG.debug("got build_urls %s" % build_urls)	×
258	except requests.HTTPError as exc:	×
259	raise BuildInfoNotFound(str(exc))	×
260	build_info = None	×
261
262	valid_builds = []	×
263	while build_urls:	×
264	some = build_urls[:max_workers]	×
265	threads = [	×
266	Thread(target=self._fetch_build_info_from_url, args=(url, i, valid_builds))
267	for i, url in enumerate(some)
268	]
269	for thread in threads:	×
270	thread.daemon = True	×
271	thread.start()	×
272	for thread in threads:	×
273	while thread.is_alive():	×
274	thread.join(0.1)	×
275	LOG.debug("got valid_builds %s" % valid_builds)	×
276	if valid_builds:	×
277	infos = sorted(valid_builds, key=lambda b: b[0])[0][1]	×
278	if fetch_txt_info:	×
279	self._update_build_info_from_txt(infos)	×
280
281	build_info = NightlyBuildInfo(	×
282	self.fetch_config,
283	build_url=infos["build_url"],
284	build_date=date,
285	changeset=infos.get("changeset"),
286	repo_url=infos.get("repository"),
287	)
288	break	×
289	build_urls = build_urls[max_workers:]	×
290
291	if build_info is None:	×
292	raise BuildInfoNotFound("Unable to find build info for %s" % date)	×
293
294	return build_info	×

mozilla / mozregression / 11327586461

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous