14615789434

Committed 23 Apr 2025 10:22AM CUT coverage: 35.122%. First build

Build # 14615789434

Build Type

Pull #1450

github

Committed by

web-flow

Commit Message

Merge 97b0a97de into 807564865

Pull Request Pull Request #1450: Bug 1763188 - Add Snap support using TC builds

Run Details

62 of 191 new or added lines in 7 files covered. (32.46%)

1054 of 3001 relevant lines covered (35.12%)

0.7 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

26.79

/mozregression/test_runner.py

"""
This module implements a :class:`TestRunner` interface for testing builds
and a default implementation :class:`ManualTestRunner`.
"""

from __future__ import absolute_import, print_function

import datetime
import os
import shlex
import subprocess
import sys
from abc import ABCMeta, abstractmethod

from mozlog import get_proxy_logger

from mozregression.errors import LauncherError, TestCommandError
from mozregression.launchers import create_launcher as mozlauncher

LOG = get_proxy_logger("Test Runner")

def create_launcher(build_info, launcher_args):
    """
    Create and returns a :class:`mozregression.launchers.Launcher`.
    """
    if build_info.build_type == "nightly":
        if isinstance(build_info.build_date, datetime.datetime):
            desc = "for buildid %s" % build_info.build_date.strftime("%Y%m%d%H%M%S")
        else:
            desc = "for %s" % build_info.build_date
    else:
        desc = "built on %s, revision %s" % (
            build_info.build_date,
            build_info.short_changeset,
        )
    LOG.info("Running %s build %s" % (build_info.repo_name, desc))

    return mozlauncher(build_info, launcher_args)


class TestRunner(metaclass=ABCMeta):
    """
    Abstract class that allows to test a build.

    :meth:`evaluate` must be implemented by subclasses.
    """

    @abstractmethod
    def evaluate(self, build_info, allow_back=False):
        """
        Evaluate a given build. Must returns a tuple of (verdict, app_info).

        The verdict must be a letter that indicate the state of the build:
        'g', 'b', 's', 'r' or 'e' respectively for 'good', 'bad', 'skip',
        'retry' or 'exit'. If **allow_back** is True, it is also possible
        to return 'back'.

        The app_info is the return value of the
        :meth:`mozregression.launchers.Launcher.get_app_info` for this
        particular build.

        :param build_path: the path to the build file to test
        :param build_info: a :class:`mozrgression.uild_info.BuildInfo` instance
        :param allow_back: indicate if the back command should be proposed.
        """
        raise NotImplementedError

    @abstractmethod
    def run_once(self, build_info):
        """
        Run the given build and wait for its completion. Return the error
        code when available.
        """
        raise NotImplementedError

    def index_to_try_after_skip(self, build_range):
        """
        Return the index of the build to use after a build was skipped.

        By default this only returns the mid point of the remaining range.
        """
        return build_range.mid_point()

    def maybe_snap(self):
        """
        Checking if the launcher migth contain Snap specific bits and return
        them if it's the case, defaulting to False else.
        """
        if hasattr(self, "launcher_kwargs") and "allow_sudo" in self.launcher_kwargs.keys():
            return (
                self.launcher_kwargs["allow_sudo"],
                self.launcher_kwargs["disable_snap_connect"],
            )
        else:
            return (False, False)


class ManualTestRunner(TestRunner):
    """
    A TestRunner subclass that run builds and ask for evaluation by
    prompting in the terminal.
    """

    def __init__(self, launcher_kwargs=None):
        TestRunner.__init__(self)
        self.launcher_kwargs = launcher_kwargs or {}

    def get_verdict(self, build_info, allow_back):
        """
        Ask and returns the verdict.
        """
        options = ["good", "bad", "skip", "retry", "exit"]
        if allow_back:
            options.insert(-1, "back")
        # allow user to just type one letter
        allowed_inputs = options + [o[0] for o in options]
        # format options to nice printing
        formatted_options = ", ".join(["'%s'" % o for o in options[:-1]]) + " or '%s'" % options[-1]
        verdict = ""
        while verdict not in allowed_inputs:
            verdict = input(
                "Was this %s build good, bad, or broken?"
                " (type %s and press Enter): " % (build_info.build_type, formatted_options)
            )

        if verdict == "back":
            return "back"
        # shorten verdict to one character for processing...
        return verdict[0]

    def evaluate(self, build_info, allow_back=False):
        with create_launcher(build_info, self.launcher_kwargs) as launcher:
            launcher.start(**self.launcher_kwargs)
            build_info.update_from_app_info(launcher.get_app_info())
            verdict = self.get_verdict(build_info, allow_back)
            try:
                launcher.stop()
            except LauncherError:
                # we got an error on process termination, but user
                # already gave the verdict, so pass this "silently"
                # (it would be logged from the launcher anyway)
                launcher._running = False
        return verdict

    def run_once(self, build_info):
        with create_launcher(
            build_info, self.launcher_kwargs
        ) as launcher:
            launcher.start(**self.launcher_kwargs)
            build_info.update_from_app_info(launcher.get_app_info())
            return launcher.wait()

    def index_to_try_after_skip(self, build_range):
        mid = TestRunner.index_to_try_after_skip(self, build_range)
        build_range_len = len(build_range)
        if build_range_len <= 3:
            # do not even ask if there is only one build to choose
            return mid
        min = -mid + 1
        max = build_range_len - mid - 2
        valid_range = list(range(min, max + 1))
        print(
            "Build was skipped. You can manually choose a new build to"
            " test, to be able to get out of a broken build range."
        )
        print(
            "Please type the index of the build you would like to try - the"
            " index is 0-based on the middle of the remaining build range."
        )
        print("You can choose a build index between [%d, %d]:" % (min, max))
        while True:
            value = input("> ")
            try:
                index = int(value)
                if index in valid_range:
                    return mid + index
            except ValueError:
                pass


def _raise_command_error(exc, msg=""):
    raise TestCommandError("Unable to run the test command%s: `%s`" % (msg, exc))


class CommandTestRunner(TestRunner):
    """
    A TestRunner subclass that evaluate builds given a shell command.

    Some variables may be used to evaluate the builds:
     - variables referenced in :meth:`TestRunner.evaluate`
     - app_name (the tested application name: firefox, ...)
     - binary (the path to the binary when applicable - not for fennec)

    These variables can be used in two ways:
    1. as environment variables. 'MOZREGRESSION_' is prepended and the
       variables names are upcased. Example: MOZREGRESSION_BINARY
    2. as placeholders in the command line. variables names must be enclosed
       with curly brackets. Example:
       `mozmill -app firefox -b {binary} -t path/to/test.js`
    """

    def __init__(self, command):
        TestRunner.__init__(self)
        self.command = command

    def evaluate(self, build_info, allow_back=False):
        with create_launcher(
            build_info, self.launcher_kwargs
        ) as launcher:
            build_info.update_from_app_info(launcher.get_app_info())
            variables = {k: v for k, v in build_info.to_dict().items()}
            if hasattr(launcher, "binary"):
                variables["binary"] = launcher.binary

            env = dict(os.environ)
            for k, v in variables.items():
                env["MOZREGRESSION_" + k.upper()] = str(v)
            try:
                command = self.command.format(**variables)
            except KeyError as exc:
                _raise_command_error(exc, " (formatting error)")
            command = os.path.expanduser(command)
            LOG.info("Running test command: `%s`" % command)

            # `shlex.split` does parsing and escaping that isn't compatible with Windows.
            if sys.platform == "win32":
                cmdlist = command
            else:
                cmdlist = shlex.split(command)

            try:
                retcode = subprocess.call(cmdlist, env=env)
            except IndexError:
                _raise_command_error("Empty command")
            except OSError as exc:
                _raise_command_error(
                    exc,
                    " (%s not found or not executable)"
                    % (command if sys.platform == "win32" else cmdlist[0]),
                )
        LOG.info(
            "Test command result: %d (build is %s)" % (retcode, "good" if retcode == 0 else "bad")
        )
        return "g" if retcode == 0 else "b"

    def run_once(self, build_info):
        return 0 if self.evaluate(build_info) == "g" else 1

1	"""
2	This module implements a :class:`TestRunner` interface for testing builds
3	and a default implementation :class:`ManualTestRunner`.
4	"""
5
6	from __future__ import absolute_import, print_function	2✔
7
8	import datetime	2✔
9	import os	2✔
10	import shlex	2✔
11	import subprocess	2✔
12	import sys	2✔
13	from abc import ABCMeta, abstractmethod	2✔
14
15	from mozlog import get_proxy_logger	2✔
16
17	from mozregression.errors import LauncherError, TestCommandError	2✔
18	from mozregression.launchers import create_launcher as mozlauncher	2✔
19
20	LOG = get_proxy_logger("Test Runner")	2✔
21
22	def create_launcher(build_info, launcher_args):	2✔
23	"""
24	Create and returns a :class:`mozregression.launchers.Launcher`.
25	"""
26	if build_info.build_type == "nightly":	×
27	if isinstance(build_info.build_date, datetime.datetime):	×
28	desc = "for buildid %s" % build_info.build_date.strftime("%Y%m%d%H%M%S")	×
29	else:
30	desc = "for %s" % build_info.build_date	×
31	else:
32	desc = "built on %s, revision %s" % (	×
33	build_info.build_date,
34	build_info.short_changeset,
35	)
36	LOG.info("Running %s build %s" % (build_info.repo_name, desc))	×
37
NEW 38	return mozlauncher(build_info, launcher_args)	×
39
40
41	class TestRunner(metaclass=ABCMeta):	2✔
42	"""
43	Abstract class that allows to test a build.
44
45	:meth:`evaluate` must be implemented by subclasses.
46	"""
47
48	@abstractmethod	2✔
49	def evaluate(self, build_info, allow_back=False):	2✔
50	"""
51	Evaluate a given build. Must returns a tuple of (verdict, app_info).
52
53	The verdict must be a letter that indicate the state of the build:
54	'g', 'b', 's', 'r' or 'e' respectively for 'good', 'bad', 'skip',
55	'retry' or 'exit'. If allow_back is True, it is also possible
56	to return 'back'.
57
58	The app_info is the return value of the
59	:meth:`mozregression.launchers.Launcher.get_app_info` for this
60	particular build.
61
62	:param build_path: the path to the build file to test
63	:param build_info: a :class:`mozrgression.uild_info.BuildInfo` instance
64	:param allow_back: indicate if the back command should be proposed.
65	"""
66	raise NotImplementedError
67
68	@abstractmethod	2✔
69	def run_once(self, build_info):	2✔
70	"""
71	Run the given build and wait for its completion. Return the error
72	code when available.
73	"""
74	raise NotImplementedError
75
76	def index_to_try_after_skip(self, build_range):	2✔
77	"""
78	Return the index of the build to use after a build was skipped.
79
80	By default this only returns the mid point of the remaining range.
81	"""
82	return build_range.mid_point()	×
83
84	def maybe_snap(self):	2✔
85	"""
86	Checking if the launcher migth contain Snap specific bits and return
87	them if it's the case, defaulting to False else.
88	"""
NEW 89	if hasattr(self, "launcher_kwargs") and "allow_sudo" in self.launcher_kwargs.keys():	×
NEW 90	return (	×
91	self.launcher_kwargs["allow_sudo"],
92	self.launcher_kwargs["disable_snap_connect"],
93	)
94	else:
NEW 95	return (False, False)	×
96
97
98	class ManualTestRunner(TestRunner):	2✔
99	"""
100	A TestRunner subclass that run builds and ask for evaluation by
101	prompting in the terminal.
102	"""
103
104	def __init__(self, launcher_kwargs=None):	2✔
105	TestRunner.__init__(self)	×
106	self.launcher_kwargs = launcher_kwargs or {}	×
107
108	def get_verdict(self, build_info, allow_back):	2✔
109	"""
110	Ask and returns the verdict.
111	"""
112	options = ["good", "bad", "skip", "retry", "exit"]	×
113	if allow_back:	×
114	options.insert(-1, "back")	×
115	# allow user to just type one letter
116	allowed_inputs = options + [o[0] for o in options]	×
117	# format options to nice printing
118	formatted_options = ", ".join(["'%s'" % o for o in options[:-1]]) + " or '%s'" % options[-1]	×
119	verdict = ""	×
120	while verdict not in allowed_inputs:	×
121	verdict = input(	×
122	"Was this %s build good, bad, or broken?"
123	" (type %s and press Enter): " % (build_info.build_type, formatted_options)
124	)
125
126	if verdict == "back":	×
127	return "back"	×
128	# shorten verdict to one character for processing...
129	return verdict[0]	×
130
131	def evaluate(self, build_info, allow_back=False):	2✔
NEW 132	with create_launcher(build_info, self.launcher_kwargs) as launcher:	×
133	launcher.start(**self.launcher_kwargs)	×
134	build_info.update_from_app_info(launcher.get_app_info())	×
135	verdict = self.get_verdict(build_info, allow_back)	×
136	try:	×
137	launcher.stop()	×
138	except LauncherError:	×
139	# we got an error on process termination, but user
140	# already gave the verdict, so pass this "silently"
141	# (it would be logged from the launcher anyway)
142	launcher._running = False	×
143	return verdict	×
144
145	def run_once(self, build_info):	2✔
NEW 146	with create_launcher(	×
147	build_info, self.launcher_kwargs
148	) as launcher:
149	launcher.start(**self.launcher_kwargs)	×
150	build_info.update_from_app_info(launcher.get_app_info())	×
151	return launcher.wait()	×
152
153	def index_to_try_after_skip(self, build_range):	2✔
154	mid = TestRunner.index_to_try_after_skip(self, build_range)	×
155	build_range_len = len(build_range)	×
156	if build_range_len <= 3:	×
157	# do not even ask if there is only one build to choose
158	return mid	×
159	min = -mid + 1	×
160	max = build_range_len - mid - 2	×
161	valid_range = list(range(min, max + 1))	×
162	print(	×
163	"Build was skipped. You can manually choose a new build to"
164	" test, to be able to get out of a broken build range."
165	)
166	print(	×
167	"Please type the index of the build you would like to try - the"
168	" index is 0-based on the middle of the remaining build range."
169	)
170	print("You can choose a build index between [%d, %d]:" % (min, max))	×
171	while True:
172	value = input("> ")	×
173	try:	×
174	index = int(value)	×
175	if index in valid_range:	×
176	return mid + index	×
177	except ValueError:	×
178	pass
179
180
181	def _raise_command_error(exc, msg=""):	2✔
182	raise TestCommandError("Unable to run the test command%s: `%s`" % (msg, exc))	×
183
184
185	class CommandTestRunner(TestRunner):	2✔
186	"""
187	A TestRunner subclass that evaluate builds given a shell command.
188
189	Some variables may be used to evaluate the builds:
190	- variables referenced in :meth:`TestRunner.evaluate`
191	- app_name (the tested application name: firefox, ...)
192	- binary (the path to the binary when applicable - not for fennec)
193
194	These variables can be used in two ways:
195	1. as environment variables. 'MOZREGRESSION_' is prepended and the
196	variables names are upcased. Example: MOZREGRESSION_BINARY
197	2. as placeholders in the command line. variables names must be enclosed
198	with curly brackets. Example:
199	`mozmill -app firefox -b {binary} -t path/to/test.js`
200	"""
201
202	def __init__(self, command):	2✔
203	TestRunner.__init__(self)	×
204	self.command = command	×
205
206	def evaluate(self, build_info, allow_back=False):	2✔
NEW 207	with create_launcher(	×
208	build_info, self.launcher_kwargs
209	) as launcher:
210	build_info.update_from_app_info(launcher.get_app_info())	×
211	variables = {k: v for k, v in build_info.to_dict().items()}	×
212	if hasattr(launcher, "binary"):	×
213	variables["binary"] = launcher.binary	×
214
215	env = dict(os.environ)	×
216	for k, v in variables.items():	×
217	env["MOZREGRESSION_" + k.upper()] = str(v)	×
218	try:	×
219	command = self.command.format(**variables)	×
220	except KeyError as exc:	×
221	_raise_command_error(exc, " (formatting error)")	×
222	command = os.path.expanduser(command)	×
223	LOG.info("Running test command: `%s`" % command)	×
224
225	# `shlex.split` does parsing and escaping that isn't compatible with Windows.
226	if sys.platform == "win32":	×
227	cmdlist = command	×
228	else:
229	cmdlist = shlex.split(command)	×
230
231	try:	×
232	retcode = subprocess.call(cmdlist, env=env)	×
233	except IndexError:	×
234	_raise_command_error("Empty command")	×
235	except OSError as exc:	×
236	_raise_command_error(	×
237	exc,
238	" (%s not found or not executable)"
239	% (command if sys.platform == "win32" else cmdlist[0]),
240	)
241	LOG.info(	×
242	"Test command result: %d (build is %s)" % (retcode, "good" if retcode == 0 else "bad")
243	)
244	return "g" if retcode == 0 else "b"	×
245
246	def run_once(self, build_info):	2✔
247	return 0 if self.evaluate(build_info) == "g" else 1	×

mozilla / mozregression / 14615789434

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous