11572580249

Committed 29 Oct 2024 11:06AM CUT coverage: 35.11%. First build

Build # 11572580249

Build Type

Pull #1450

github

Committed by

web-flow

Commit Message

Merge 25ca7f46b into f558f7daa

Pull Request Pull Request #1450: Bug 1763188 - Add Snap support using TC builds

Run Details

62 of 193 new or added lines in 7 files covered. (32.12%)

1054 of 3002 relevant lines covered (35.11%)

1.05 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

26.09

/mozregression/test_runner.py

"""
This module implements a :class:`TestRunner` interface for testing builds
and a default implementation :class:`ManualTestRunner`.
"""

from __future__ import absolute_import, print_function

import datetime
import os
import shlex
import subprocess
import sys
from abc import ABCMeta, abstractmethod

from mozlog import get_proxy_logger

from mozregression.errors import LauncherError, TestCommandError
from mozregression.launchers import create_launcher as mozlauncher

LOG = get_proxy_logger("Test Runner")


def create_launcher(build_info, allow_sudo=False, disable_snap_connect=False):
    """
    Create and returns a :class:`mozregression.launchers.Launcher`.
    """
    if build_info.build_type == "nightly":
        if isinstance(build_info.build_date, datetime.datetime):
            desc = "for buildid %s" % build_info.build_date.strftime("%Y%m%d%H%M%S")
        else:
            desc = "for %s" % build_info.build_date
    else:
        desc = "built on %s, revision %s" % (
            build_info.build_date,
            build_info.short_changeset,
        )
    LOG.info("Running %s build %s" % (build_info.repo_name, desc))

    return mozlauncher(build_info, allow_sudo, disable_snap_connect)


class TestRunner(metaclass=ABCMeta):
    """
    Abstract class that allows to test a build.

    :meth:`evaluate` must be implemented by subclasses.
    """

    @abstractmethod
    def evaluate(self, build_info, allow_back=False):
        """
        Evaluate a given build. Must returns a tuple of (verdict, app_info).

        The verdict must be a letter that indicate the state of the build:
        'g', 'b', 's', 'r' or 'e' respectively for 'good', 'bad', 'skip',
        'retry' or 'exit'. If **allow_back** is True, it is also possible
        to return 'back'.

        The app_info is the return value of the
        :meth:`mozregression.launchers.Launcher.get_app_info` for this
        particular build.

        :param build_path: the path to the build file to test
        :param build_info: a :class:`mozrgression.uild_info.BuildInfo` instance
        :param allow_back: indicate if the back command should be proposed.
        """
        raise NotImplementedError

    @abstractmethod
    def run_once(self, build_info):
        """
        Run the given build and wait for its completion. Return the error
        code when available.
        """
        raise NotImplementedError

    def index_to_try_after_skip(self, build_range):
        """
        Return the index of the build to use after a build was skipped.

        By default this only returns the mid point of the remaining range.
        """
        return build_range.mid_point()

    def maybe_snap(self):
        """
        Checking if the launcher migth contain Snap specific bits and return
        them if it's the case, defaulting to False else.
        """
        if hasattr(self, "launcher_kwargs") and "allow_sudo" in self.launcher_kwargs.keys():
            return (
                self.launcher_kwargs["allow_sudo"],
                self.launcher_kwargs["disable_snap_connect"],
            )
        else:
            return (False, False)


class ManualTestRunner(TestRunner):
    """
    A TestRunner subclass that run builds and ask for evaluation by
    prompting in the terminal.
    """

    def __init__(self, launcher_kwargs=None):
        TestRunner.__init__(self)
        self.launcher_kwargs = launcher_kwargs or {}

    def get_verdict(self, build_info, allow_back):
        """
        Ask and returns the verdict.
        """
        options = ["good", "bad", "skip", "retry", "exit"]
        if allow_back:
            options.insert(-1, "back")
        # allow user to just type one letter
        allowed_inputs = options + [o[0] for o in options]
        # format options to nice printing
        formatted_options = ", ".join(["'%s'" % o for o in options[:-1]]) + " or '%s'" % options[-1]
        verdict = ""
        while verdict not in allowed_inputs:
            verdict = input(
                "Was this %s build good, bad, or broken?"
                " (type %s and press Enter): " % (build_info.build_type, formatted_options)
            )

        if verdict == "back":
            return "back"
        # shorten verdict to one character for processing...
        return verdict[0]

    def evaluate(self, build_info, allow_back=False):
        (allow_sudo, disable_snap_connect) = self.maybe_snap()
        with create_launcher(build_info, allow_sudo, disable_snap_connect) as launcher:
            launcher.start(**self.launcher_kwargs)
            build_info.update_from_app_info(launcher.get_app_info())
            verdict = self.get_verdict(build_info, allow_back)
            try:
                launcher.stop()
            except LauncherError:
                # we got an error on process termination, but user
                # already gave the verdict, so pass this "silently"
                # (it would be logged from the launcher anyway)
                launcher._running = False
        return verdict

    def run_once(self, build_info):
        (allow_sudo, disable_snap_connect) = self.maybe_snap()
        with create_launcher(
            build_info,
            allow_sudo,
            disable_snap_connect,
        ) as launcher:
            launcher.start(**self.launcher_kwargs)
            build_info.update_from_app_info(launcher.get_app_info())
            return launcher.wait()

    def index_to_try_after_skip(self, build_range):
        mid = TestRunner.index_to_try_after_skip(self, build_range)
        build_range_len = len(build_range)
        if build_range_len <= 3:
            # do not even ask if there is only one build to choose
            return mid
        min = -mid + 1
        max = build_range_len - mid - 2
        valid_range = list(range(min, max + 1))
        print(
            "Build was skipped. You can manually choose a new build to"
            " test, to be able to get out of a broken build range."
        )
        print(
            "Please type the index of the build you would like to try - the"
            " index is 0-based on the middle of the remaining build range."
        )
        print("You can choose a build index between [%d, %d]:" % (min, max))
        while True:
            value = input("> ")
            try:
                index = int(value)
                if index in valid_range:
                    return mid + index
            except ValueError:
                pass


def _raise_command_error(exc, msg=""):
    raise TestCommandError("Unable to run the test command%s: `%s`" % (msg, exc))


class CommandTestRunner(TestRunner):
    """
    A TestRunner subclass that evaluate builds given a shell command.

    Some variables may be used to evaluate the builds:
     - variables referenced in :meth:`TestRunner.evaluate`
     - app_name (the tested application name: firefox, ...)
     - binary (the path to the binary when applicable - not for fennec)

    These variables can be used in two ways:
    1. as environment variables. 'MOZREGRESSION_' is prepended and the
       variables names are upcased. Example: MOZREGRESSION_BINARY
    2. as placeholders in the command line. variables names must be enclosed
       with curly brackets. Example:
       `mozmill -app firefox -b {binary} -t path/to/test.js`
    """

    def __init__(self, command):
        TestRunner.__init__(self)
        self.command = command

    def evaluate(self, build_info, allow_back=False):
        (allow_sudo, disable_snap_connect) = self.maybe_snap()
        with create_launcher(
            build_info,
            allow_sudo,
            disable_snap_connect,
        ) as launcher:
            build_info.update_from_app_info(launcher.get_app_info())
            variables = {k: v for k, v in build_info.to_dict().items()}
            if hasattr(launcher, "binary"):
                variables["binary"] = launcher.binary

            env = dict(os.environ)
            for k, v in variables.items():
                env["MOZREGRESSION_" + k.upper()] = str(v)
            try:
                command = self.command.format(**variables)
            except KeyError as exc:
                _raise_command_error(exc, " (formatting error)")
            command = os.path.expanduser(command)
            LOG.info("Running test command: `%s`" % command)

            # `shlex.split` does parsing and escaping that isn't compatible with Windows.
            if sys.platform == "win32":
                cmdlist = command
            else:
                cmdlist = shlex.split(command)

            try:
                retcode = subprocess.call(cmdlist, env=env)
            except IndexError:
                _raise_command_error("Empty command")
            except OSError as exc:
                _raise_command_error(
                    exc,
                    " (%s not found or not executable)"
                    % (command if sys.platform == "win32" else cmdlist[0]),
                )
        LOG.info(
            "Test command result: %d (build is %s)" % (retcode, "good" if retcode == 0 else "bad")
        )
        return "g" if retcode == 0 else "b"

    def run_once(self, build_info):
        return 0 if self.evaluate(build_info) == "g" else 1

1	"""
2	This module implements a :class:`TestRunner` interface for testing builds
3	and a default implementation :class:`ManualTestRunner`.
4	"""
5
6	from __future__ import absolute_import, print_function	3✔
7
8	import datetime	3✔
9	import os	3✔
10	import shlex	3✔
11	import subprocess	3✔
12	import sys	3✔
13	from abc import ABCMeta, abstractmethod	3✔
14
15	from mozlog import get_proxy_logger	3✔
16
17	from mozregression.errors import LauncherError, TestCommandError	3✔
18	from mozregression.launchers import create_launcher as mozlauncher	3✔
19
20	LOG = get_proxy_logger("Test Runner")	3✔
21
22
23	def create_launcher(build_info, allow_sudo=False, disable_snap_connect=False):	3✔
24	"""
25	Create and returns a :class:`mozregression.launchers.Launcher`.
26	"""
27	if build_info.build_type == "nightly":	×
28	if isinstance(build_info.build_date, datetime.datetime):	×
29	desc = "for buildid %s" % build_info.build_date.strftime("%Y%m%d%H%M%S")	×
30	else:
31	desc = "for %s" % build_info.build_date	×
32	else:
33	desc = "built on %s, revision %s" % (	×
34	build_info.build_date,
35	build_info.short_changeset,
36	)
37	LOG.info("Running %s build %s" % (build_info.repo_name, desc))	×
38
NEW 39	return mozlauncher(build_info, allow_sudo, disable_snap_connect)	×
40
41
42	class TestRunner(metaclass=ABCMeta):	3✔
43	"""
44	Abstract class that allows to test a build.
45
46	:meth:`evaluate` must be implemented by subclasses.
47	"""
48
49	@abstractmethod	3✔
50	def evaluate(self, build_info, allow_back=False):	3✔
51	"""
52	Evaluate a given build. Must returns a tuple of (verdict, app_info).
53
54	The verdict must be a letter that indicate the state of the build:
55	'g', 'b', 's', 'r' or 'e' respectively for 'good', 'bad', 'skip',
56	'retry' or 'exit'. If allow_back is True, it is also possible
57	to return 'back'.
58
59	The app_info is the return value of the
60	:meth:`mozregression.launchers.Launcher.get_app_info` for this
61	particular build.
62
63	:param build_path: the path to the build file to test
64	:param build_info: a :class:`mozrgression.uild_info.BuildInfo` instance
65	:param allow_back: indicate if the back command should be proposed.
66	"""
67	raise NotImplementedError
68
69	@abstractmethod	3✔
70	def run_once(self, build_info):	3✔
71	"""
72	Run the given build and wait for its completion. Return the error
73	code when available.
74	"""
75	raise NotImplementedError
76
77	def index_to_try_after_skip(self, build_range):	3✔
78	"""
79	Return the index of the build to use after a build was skipped.
80
81	By default this only returns the mid point of the remaining range.
82	"""
83	return build_range.mid_point()	×
84
85	def maybe_snap(self):	3✔
86	"""
87	Checking if the launcher migth contain Snap specific bits and return
88	them if it's the case, defaulting to False else.
89	"""
NEW 90	if hasattr(self, "launcher_kwargs") and "allow_sudo" in self.launcher_kwargs.keys():	×
NEW 91	return (	×
92	self.launcher_kwargs["allow_sudo"],
93	self.launcher_kwargs["disable_snap_connect"],
94	)
95	else:
NEW 96	return (False, False)	×
97
98
99	class ManualTestRunner(TestRunner):	3✔
100	"""
101	A TestRunner subclass that run builds and ask for evaluation by
102	prompting in the terminal.
103	"""
104
105	def __init__(self, launcher_kwargs=None):	3✔
106	TestRunner.__init__(self)	×
107	self.launcher_kwargs = launcher_kwargs or {}	×
108
109	def get_verdict(self, build_info, allow_back):	3✔
110	"""
111	Ask and returns the verdict.
112	"""
113	options = ["good", "bad", "skip", "retry", "exit"]	×
114	if allow_back:	×
115	options.insert(-1, "back")	×
116	# allow user to just type one letter
117	allowed_inputs = options + [o[0] for o in options]	×
118	# format options to nice printing
119	formatted_options = ", ".join(["'%s'" % o for o in options[:-1]]) + " or '%s'" % options[-1]	×
120	verdict = ""	×
121	while verdict not in allowed_inputs:	×
122	verdict = input(	×
123	"Was this %s build good, bad, or broken?"
124	" (type %s and press Enter): " % (build_info.build_type, formatted_options)
125	)
126
127	if verdict == "back":	×
128	return "back"	×
129	# shorten verdict to one character for processing...
130	return verdict[0]	×
131
132	def evaluate(self, build_info, allow_back=False):	3✔
NEW 133	(allow_sudo, disable_snap_connect) = self.maybe_snap()	×
NEW 134	with create_launcher(build_info, allow_sudo, disable_snap_connect) as launcher:	×
135	launcher.start(**self.launcher_kwargs)	×
136	build_info.update_from_app_info(launcher.get_app_info())	×
137	verdict = self.get_verdict(build_info, allow_back)	×
138	try:	×
139	launcher.stop()	×
140	except LauncherError:	×
141	# we got an error on process termination, but user
142	# already gave the verdict, so pass this "silently"
143	# (it would be logged from the launcher anyway)
144	launcher._running = False	×
145	return verdict	×
146
147	def run_once(self, build_info):	3✔
NEW 148	(allow_sudo, disable_snap_connect) = self.maybe_snap()	×
NEW 149	with create_launcher(	×
150	build_info,
151	allow_sudo,
152	disable_snap_connect,
153	) as launcher:
154	launcher.start(**self.launcher_kwargs)	×
155	build_info.update_from_app_info(launcher.get_app_info())	×
156	return launcher.wait()	×
157
158	def index_to_try_after_skip(self, build_range):	3✔
159	mid = TestRunner.index_to_try_after_skip(self, build_range)	×
160	build_range_len = len(build_range)	×
161	if build_range_len <= 3:	×
162	# do not even ask if there is only one build to choose
163	return mid	×
164	min = -mid + 1	×
165	max = build_range_len - mid - 2	×
166	valid_range = list(range(min, max + 1))	×
167	print(	×
168	"Build was skipped. You can manually choose a new build to"
169	" test, to be able to get out of a broken build range."
170	)
171	print(	×
172	"Please type the index of the build you would like to try - the"
173	" index is 0-based on the middle of the remaining build range."
174	)
175	print("You can choose a build index between [%d, %d]:" % (min, max))	×
176	while True:
177	value = input("> ")	×
178	try:	×
179	index = int(value)	×
180	if index in valid_range:	×
181	return mid + index	×
182	except ValueError:	×
183	pass
184
185
186	def _raise_command_error(exc, msg=""):	3✔
187	raise TestCommandError("Unable to run the test command%s: `%s`" % (msg, exc))	×
188
189
190	class CommandTestRunner(TestRunner):	3✔
191	"""
192	A TestRunner subclass that evaluate builds given a shell command.
193
194	Some variables may be used to evaluate the builds:
195	- variables referenced in :meth:`TestRunner.evaluate`
196	- app_name (the tested application name: firefox, ...)
197	- binary (the path to the binary when applicable - not for fennec)
198
199	These variables can be used in two ways:
200	1. as environment variables. 'MOZREGRESSION_' is prepended and the
201	variables names are upcased. Example: MOZREGRESSION_BINARY
202	2. as placeholders in the command line. variables names must be enclosed
203	with curly brackets. Example:
204	`mozmill -app firefox -b {binary} -t path/to/test.js`
205	"""
206
207	def __init__(self, command):	3✔
208	TestRunner.__init__(self)	×
209	self.command = command	×
210
211	def evaluate(self, build_info, allow_back=False):	3✔
NEW 212	(allow_sudo, disable_snap_connect) = self.maybe_snap()	×
NEW 213	with create_launcher(	×
214	build_info,
215	allow_sudo,
216	disable_snap_connect,
217	) as launcher:
218	build_info.update_from_app_info(launcher.get_app_info())	×
219	variables = {k: v for k, v in build_info.to_dict().items()}	×
220	if hasattr(launcher, "binary"):	×
221	variables["binary"] = launcher.binary	×
222
223	env = dict(os.environ)	×
224	for k, v in variables.items():	×
225	env["MOZREGRESSION_" + k.upper()] = str(v)	×
226	try:	×
227	command = self.command.format(**variables)	×
228	except KeyError as exc:	×
229	_raise_command_error(exc, " (formatting error)")	×
230	command = os.path.expanduser(command)	×
231	LOG.info("Running test command: `%s`" % command)	×
232
233	# `shlex.split` does parsing and escaping that isn't compatible with Windows.
234	if sys.platform == "win32":	×
235	cmdlist = command	×
236	else:
237	cmdlist = shlex.split(command)	×
238
239	try:	×
240	retcode = subprocess.call(cmdlist, env=env)	×
241	except IndexError:	×
242	_raise_command_error("Empty command")	×
243	except OSError as exc:	×
244	_raise_command_error(	×
245	exc,
246	" (%s not found or not executable)"
247	% (command if sys.platform == "win32" else cmdlist[0]),
248	)
249	LOG.info(	×
250	"Test command result: %d (build is %s)" % (retcode, "good" if retcode == 0 else "bad")
251	)
252	return "g" if retcode == 0 else "b"	×
253
254	def run_once(self, build_info):	3✔
255	return 0 if self.evaluate(build_info) == "g" else 1	×

mozilla / mozregression / 11572580249

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous