14620593583

Committed 23 Apr 2025 02:22PM UTC coverage: 87.419%. First build

Build # 14620593583

Build Type

Pull #1983

github

Committed by

web-flow

Commit Message

Merge bef531d5f into 807564865

Pull Request Pull Request #1983: Bug 1763188 - Add Snap support using TC builds

Run Details

60 of 133 new or added lines in 4 files covered. (45.11%)

2578 of 2949 relevant lines covered (87.42%)

8.54 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

96.46

/mozregression/test_runner.py

"""
This module implements a :class:`TestRunner` interface for testing builds
and a default implementation :class:`ManualTestRunner`.
"""

from __future__ import absolute_import, print_function

import datetime
import os
import shlex
import subprocess
import sys
from abc import ABCMeta, abstractmethod

from mozlog import get_proxy_logger

from mozregression.errors import LauncherError, TestCommandError
from mozregression.launchers import create_launcher as mozlauncher

LOG = get_proxy_logger("Test Runner")


def create_launcher(build_info, launcher_args):
    """
    Create and returns a :class:`mozregression.launchers.Launcher`.
    """
    if build_info.build_type == "nightly":
        if isinstance(build_info.build_date, datetime.datetime):
            desc = "for buildid %s" % build_info.build_date.strftime("%Y%m%d%H%M%S")
        else:
            desc = "for %s" % build_info.build_date
    else:
        desc = "built on %s, revision %s" % (
            build_info.build_date,
            build_info.short_changeset,
        )
    LOG.info("Running %s build %s" % (build_info.repo_name, desc))

    return mozlauncher(build_info, launcher_args)


class TestRunner(metaclass=ABCMeta):
    """
    Abstract class that allows to test a build.

    :meth:`evaluate` must be implemented by subclasses.
    """

    @abstractmethod
    def evaluate(self, build_info, allow_back=False):
        """
        Evaluate a given build. Must returns a tuple of (verdict, app_info).

        The verdict must be a letter that indicate the state of the build:
        'g', 'b', 's', 'r' or 'e' respectively for 'good', 'bad', 'skip',
        'retry' or 'exit'. If **allow_back** is True, it is also possible
        to return 'back'.

        The app_info is the return value of the
        :meth:`mozregression.launchers.Launcher.get_app_info` for this
        particular build.

        :param build_path: the path to the build file to test
        :param build_info: a :class:`mozrgression.uild_info.BuildInfo` instance
        :param allow_back: indicate if the back command should be proposed.
        """
        raise NotImplementedError

    @abstractmethod
    def run_once(self, build_info):
        """
        Run the given build and wait for its completion. Return the error
        code when available.
        """
        raise NotImplementedError

    def index_to_try_after_skip(self, build_range):
        """
        Return the index of the build to use after a build was skipped.

        By default this only returns the mid point of the remaining range.
        """
        return build_range.mid_point()

    def maybe_snap(self):
        """
        Checking if the launcher migth contain Snap specific bits and return
        them if it's the case, defaulting to False else.
        """
        if hasattr(self, "launcher_kwargs") and "allow_sudo" in self.launcher_kwargs.keys():
            return (
                self.launcher_kwargs["allow_sudo"],
                self.launcher_kwargs["disable_snap_connect"],
            )
        else:
            return (False, False)


class ManualTestRunner(TestRunner):
    """
    A TestRunner subclass that run builds and ask for evaluation by
    prompting in the terminal.
    """

    def __init__(self, launcher_kwargs=None):
        TestRunner.__init__(self)
        self.launcher_kwargs = launcher_kwargs or {}

    def get_verdict(self, build_info, allow_back):
        """
        Ask and returns the verdict.
        """
        options = ["good", "bad", "skip", "retry", "exit"]
        if allow_back:
            options.insert(-1, "back")
        # allow user to just type one letter
        allowed_inputs = options + [o[0] for o in options]
        # format options to nice printing
        formatted_options = ", ".join(["'%s'" % o for o in options[:-1]]) + " or '%s'" % options[-1]
        verdict = ""
        while verdict not in allowed_inputs:
            verdict = input(
                "Was this %s build good, bad, or broken?"
                " (type %s and press Enter): " % (build_info.build_type, formatted_options)
            )

        if verdict == "back":
            return "back"
        # shorten verdict to one character for processing...
        return verdict[0]

    def evaluate(self, build_info, allow_back=False):
        with create_launcher(build_info, self.launcher_kwargs) as launcher:
            launcher.start(**self.launcher_kwargs)
            build_info.update_from_app_info(launcher.get_app_info())
            verdict = self.get_verdict(build_info, allow_back)
            try:
                launcher.stop()
            except LauncherError:
                # we got an error on process termination, but user
                # already gave the verdict, so pass this "silently"
                # (it would be logged from the launcher anyway)
                launcher._running = False
        return verdict

    def run_once(self, build_info):
        with create_launcher(build_info, self.launcher_kwargs) as launcher:
            launcher.start(**self.launcher_kwargs)
            build_info.update_from_app_info(launcher.get_app_info())
            return launcher.wait()

    def index_to_try_after_skip(self, build_range):
        mid = TestRunner.index_to_try_after_skip(self, build_range)
        build_range_len = len(build_range)
        if build_range_len <= 3:
            # do not even ask if there is only one build to choose
            return mid
        min = -mid + 1
        max = build_range_len - mid - 2
        valid_range = list(range(min, max + 1))
        print(
            "Build was skipped. You can manually choose a new build to"
            " test, to be able to get out of a broken build range."
        )
        print(
            "Please type the index of the build you would like to try - the"
            " index is 0-based on the middle of the remaining build range."
        )
        print("You can choose a build index between [%d, %d]:" % (min, max))
        while True:
            value = input("> ")
            try:
                index = int(value)
                if index in valid_range:
                    return mid + index
            except ValueError:
                pass


def _raise_command_error(exc, msg=""):
    raise TestCommandError("Unable to run the test command%s: `%s`" % (msg, exc))


class CommandTestRunner(TestRunner):
    """
    A TestRunner subclass that evaluate builds given a shell command.

    Some variables may be used to evaluate the builds:
     - variables referenced in :meth:`TestRunner.evaluate`
     - app_name (the tested application name: firefox, ...)
     - binary (the path to the binary when applicable - not for fennec)

    These variables can be used in two ways:
    1. as environment variables. 'MOZREGRESSION_' is prepended and the
       variables names are upcased. Example: MOZREGRESSION_BINARY
    2. as placeholders in the command line. variables names must be enclosed
       with curly brackets. Example:
       `mozmill -app firefox -b {binary} -t path/to/test.js`
    """

    def __init__(self, command):
        TestRunner.__init__(self)
        self.command = command

    def evaluate(self, build_info, allow_back=False):
        with create_launcher(build_info) as launcher:
            build_info.update_from_app_info(launcher.get_app_info())
            variables = {k: v for k, v in build_info.to_dict().items()}
            if hasattr(launcher, "binary"):
                variables["binary"] = launcher.binary

            env = dict(os.environ)
            for k, v in variables.items():
                env["MOZREGRESSION_" + k.upper()] = str(v)
            try:
                command = self.command.format(**variables)
            except KeyError as exc:
                _raise_command_error(exc, " (formatting error)")
            command = os.path.expanduser(command)
            LOG.info("Running test command: `%s`" % command)

            # `shlex.split` does parsing and escaping that isn't compatible with Windows.
            if sys.platform == "win32":
                cmdlist = command
            else:
                cmdlist = shlex.split(command)

            try:
                retcode = subprocess.call(cmdlist, env=env)
            except IndexError:
                _raise_command_error("Empty command")
            except OSError as exc:
                _raise_command_error(
                    exc,
                    " (%s not found or not executable)"
                    % (command if sys.platform == "win32" else cmdlist[0]),
                )
        LOG.info(
            "Test command result: %d (build is %s)" % (retcode, "good" if retcode == 0 else "bad")
        )
        return "g" if retcode == 0 else "b"

    def run_once(self, build_info):
        return 0 if self.evaluate(build_info) == "g" else 1

1	"""
2	This module implements a :class:`TestRunner` interface for testing builds
3	and a default implementation :class:`ManualTestRunner`.
4	"""
5
6	from __future__ import absolute_import, print_function	11✔
7
8	import datetime	11✔
9	import os	11✔
10	import shlex	11✔
11	import subprocess	11✔
12	import sys	11✔
13	from abc import ABCMeta, abstractmethod	11✔
14
15	from mozlog import get_proxy_logger	11✔
16
17	from mozregression.errors import LauncherError, TestCommandError	11✔
18	from mozregression.launchers import create_launcher as mozlauncher	11✔
19
20	LOG = get_proxy_logger("Test Runner")	11✔
21
22
23	def create_launcher(build_info, launcher_args):	11✔
24	"""
25	Create and returns a :class:`mozregression.launchers.Launcher`.
26	"""
27	if build_info.build_type == "nightly":	9✔
28	if isinstance(build_info.build_date, datetime.datetime):	9✔
29	desc = "for buildid %s" % build_info.build_date.strftime("%Y%m%d%H%M%S")	9✔
30	else:
31	desc = "for %s" % build_info.build_date	9✔
32	else:
33	desc = "built on %s, revision %s" % (	9✔
34	build_info.build_date,
35	build_info.short_changeset,
36	)
37	LOG.info("Running %s build %s" % (build_info.repo_name, desc))	9✔
38
39	return mozlauncher(build_info, launcher_args)	9✔
40
41
42	class TestRunner(metaclass=ABCMeta):	11✔
43	"""
44	Abstract class that allows to test a build.
45
46	:meth:`evaluate` must be implemented by subclasses.
47	"""
48
49	@abstractmethod	11✔
50	def evaluate(self, build_info, allow_back=False):	11✔
51	"""
52	Evaluate a given build. Must returns a tuple of (verdict, app_info).
53
54	The verdict must be a letter that indicate the state of the build:
55	'g', 'b', 's', 'r' or 'e' respectively for 'good', 'bad', 'skip',
56	'retry' or 'exit'. If allow_back is True, it is also possible
57	to return 'back'.
58
59	The app_info is the return value of the
60	:meth:`mozregression.launchers.Launcher.get_app_info` for this
61	particular build.
62
63	:param build_path: the path to the build file to test
64	:param build_info: a :class:`mozrgression.uild_info.BuildInfo` instance
65	:param allow_back: indicate if the back command should be proposed.
66	"""
67	raise NotImplementedError
68
69	@abstractmethod	11✔
70	def run_once(self, build_info):	11✔
71	"""
72	Run the given build and wait for its completion. Return the error
73	code when available.
74	"""
75	raise NotImplementedError
76
77	def index_to_try_after_skip(self, build_range):	11✔
78	"""
79	Return the index of the build to use after a build was skipped.
80
81	By default this only returns the mid point of the remaining range.
82	"""
83	return build_range.mid_point()	9✔
84
85	def maybe_snap(self):	11✔
86	"""
87	Checking if the launcher migth contain Snap specific bits and return
88	them if it's the case, defaulting to False else.
89	"""
NEW 90	if hasattr(self, "launcher_kwargs") and "allow_sudo" in self.launcher_kwargs.keys():	×
NEW 91	return (	×
92	self.launcher_kwargs["allow_sudo"],
93	self.launcher_kwargs["disable_snap_connect"],
94	)
95	else:
NEW 96	return (False, False)	×
97
98
99	class ManualTestRunner(TestRunner):	11✔
100	"""
101	A TestRunner subclass that run builds and ask for evaluation by
102	prompting in the terminal.
103	"""
104
105	def __init__(self, launcher_kwargs=None):	11✔
106	TestRunner.__init__(self)	9✔
107	self.launcher_kwargs = launcher_kwargs or {}	9✔
108
109	def get_verdict(self, build_info, allow_back):	11✔
110	"""
111	Ask and returns the verdict.
112	"""
113	options = ["good", "bad", "skip", "retry", "exit"]	9✔
114	if allow_back:	9✔
115	options.insert(-1, "back")	9✔
116	# allow user to just type one letter
117	allowed_inputs = options + [o[0] for o in options]	9✔
118	# format options to nice printing
119	formatted_options = ", ".join(["'%s'" % o for o in options[:-1]]) + " or '%s'" % options[-1]	9✔
120	verdict = ""	9✔
121	while verdict not in allowed_inputs:	9✔
122	verdict = input(	9✔
123	"Was this %s build good, bad, or broken?"
124	" (type %s and press Enter): " % (build_info.build_type, formatted_options)
125	)
126
127	if verdict == "back":	9✔
128	return "back"	9✔
129	# shorten verdict to one character for processing...
130	return verdict[0]	9✔
131
132	def evaluate(self, build_info, allow_back=False):	11✔
133	with create_launcher(build_info, self.launcher_kwargs) as launcher:	9✔
134	launcher.start(**self.launcher_kwargs)	9✔
135	build_info.update_from_app_info(launcher.get_app_info())	9✔
136	verdict = self.get_verdict(build_info, allow_back)	9✔
137	try:	9✔
138	launcher.stop()	9✔
139	except LauncherError:	9✔
140	# we got an error on process termination, but user
141	# already gave the verdict, so pass this "silently"
142	# (it would be logged from the launcher anyway)
143	launcher._running = False	9✔
144	return verdict	9✔
145
146	def run_once(self, build_info):	11✔
147	with create_launcher(build_info, self.launcher_kwargs) as launcher:	9✔
148	launcher.start(**self.launcher_kwargs)	9✔
149	build_info.update_from_app_info(launcher.get_app_info())	9✔
150	return launcher.wait()	9✔
151
152	def index_to_try_after_skip(self, build_range):	11✔
153	mid = TestRunner.index_to_try_after_skip(self, build_range)	9✔
154	build_range_len = len(build_range)	9✔
155	if build_range_len <= 3:	9✔
156	# do not even ask if there is only one build to choose
157	return mid	9✔
158	min = -mid + 1	9✔
159	max = build_range_len - mid - 2	9✔
160	valid_range = list(range(min, max + 1))	9✔
161	print(	9✔
162	"Build was skipped. You can manually choose a new build to"
163	" test, to be able to get out of a broken build range."
164	)
165	print(	9✔
166	"Please type the index of the build you would like to try - the"
167	" index is 0-based on the middle of the remaining build range."
168	)
169	print("You can choose a build index between [%d, %d]:" % (min, max))	9✔
170	while True:	6✔
171	value = input("> ")	9✔
172	try:	9✔
173	index = int(value)	9✔
174	if index in valid_range:	9✔
175	return mid + index	9✔
176	except ValueError:	9✔
177	pass
178
179
180	def _raise_command_error(exc, msg=""):	11✔
181	raise TestCommandError("Unable to run the test command%s: `%s`" % (msg, exc))	9✔
182
183
184	class CommandTestRunner(TestRunner):	11✔
185	"""
186	A TestRunner subclass that evaluate builds given a shell command.
187
188	Some variables may be used to evaluate the builds:
189	- variables referenced in :meth:`TestRunner.evaluate`
190	- app_name (the tested application name: firefox, ...)
191	- binary (the path to the binary when applicable - not for fennec)
192
193	These variables can be used in two ways:
194	1. as environment variables. 'MOZREGRESSION_' is prepended and the
195	variables names are upcased. Example: MOZREGRESSION_BINARY
196	2. as placeholders in the command line. variables names must be enclosed
197	with curly brackets. Example:
198	`mozmill -app firefox -b {binary} -t path/to/test.js`
199	"""
200
201	def __init__(self, command):	11✔
202	TestRunner.__init__(self)	9✔
203	self.command = command	9✔
204
205	def evaluate(self, build_info, allow_back=False):	11✔
206	with create_launcher(build_info) as launcher:	9✔
207	build_info.update_from_app_info(launcher.get_app_info())	9✔
208	variables = {k: v for k, v in build_info.to_dict().items()}	9✔
209	if hasattr(launcher, "binary"):	9✔
210	variables["binary"] = launcher.binary	9✔
211
212	env = dict(os.environ)	9✔
213	for k, v in variables.items():	9✔
214	env["MOZREGRESSION_" + k.upper()] = str(v)	9✔
215	try:	9✔
216	command = self.command.format(**variables)	9✔
217	except KeyError as exc:	9✔
218	_raise_command_error(exc, " (formatting error)")	9✔
219	command = os.path.expanduser(command)	9✔
220	LOG.info("Running test command: `%s`" % command)	9✔
221
222	# `shlex.split` does parsing and escaping that isn't compatible with Windows.
223	if sys.platform == "win32":	9✔
224	cmdlist = command	×
225	else:
226	cmdlist = shlex.split(command)	9✔
227
228	try:	9✔
229	retcode = subprocess.call(cmdlist, env=env)	9✔
230	except IndexError:	9✔
231	_raise_command_error("Empty command")	9✔
232	except OSError as exc:	9✔
233	_raise_command_error(	9✔
234	exc,
235	" (%s not found or not executable)"
236	% (command if sys.platform == "win32" else cmdlist[0]),
237	)
238	LOG.info(	9✔
239	"Test command result: %d (build is %s)" % (retcode, "good" if retcode == 0 else "bad")
240	)
241	return "g" if retcode == 0 else "b"	9✔
242
243	def run_once(self, build_info):	11✔
244	return 0 if self.evaluate(build_info) == "g" else 1	9✔

mozilla / mozregression / 14620593583

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous