• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

mozilla / mozregression / 14615789434

23 Apr 2025 10:22AM CUT coverage: 35.122%. First build
14615789434

Pull #1450

github

web-flow
Merge 97b0a97de into 807564865
Pull Request #1450: Bug 1763188 - Add Snap support using TC builds

62 of 191 new or added lines in 7 files covered. (32.46%)

1054 of 3001 relevant lines covered (35.12%)

0.7 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

26.79
/mozregression/test_runner.py
1
"""
2
This module implements a :class:`TestRunner` interface for testing builds
3
and a default implementation :class:`ManualTestRunner`.
4
"""
5

6
from __future__ import absolute_import, print_function
2✔
7

8
import datetime
2✔
9
import os
2✔
10
import shlex
2✔
11
import subprocess
2✔
12
import sys
2✔
13
from abc import ABCMeta, abstractmethod
2✔
14

15
from mozlog import get_proxy_logger
2✔
16

17
from mozregression.errors import LauncherError, TestCommandError
2✔
18
from mozregression.launchers import create_launcher as mozlauncher
2✔
19

20
LOG = get_proxy_logger("Test Runner")
2✔
21

22
def create_launcher(build_info, launcher_args):
2✔
23
    """
24
    Create and returns a :class:`mozregression.launchers.Launcher`.
25
    """
26
    if build_info.build_type == "nightly":
×
27
        if isinstance(build_info.build_date, datetime.datetime):
×
28
            desc = "for buildid %s" % build_info.build_date.strftime("%Y%m%d%H%M%S")
×
29
        else:
30
            desc = "for %s" % build_info.build_date
×
31
    else:
32
        desc = "built on %s, revision %s" % (
×
33
            build_info.build_date,
34
            build_info.short_changeset,
35
        )
36
    LOG.info("Running %s build %s" % (build_info.repo_name, desc))
×
37

NEW
38
    return mozlauncher(build_info, launcher_args)
×
39

40

41
class TestRunner(metaclass=ABCMeta):
2✔
42
    """
43
    Abstract class that allows to test a build.
44

45
    :meth:`evaluate` must be implemented by subclasses.
46
    """
47

48
    @abstractmethod
2✔
49
    def evaluate(self, build_info, allow_back=False):
2✔
50
        """
51
        Evaluate a given build. Must returns a tuple of (verdict, app_info).
52

53
        The verdict must be a letter that indicate the state of the build:
54
        'g', 'b', 's', 'r' or 'e' respectively for 'good', 'bad', 'skip',
55
        'retry' or 'exit'. If **allow_back** is True, it is also possible
56
        to return 'back'.
57

58
        The app_info is the return value of the
59
        :meth:`mozregression.launchers.Launcher.get_app_info` for this
60
        particular build.
61

62
        :param build_path: the path to the build file to test
63
        :param build_info: a :class:`mozrgression.uild_info.BuildInfo` instance
64
        :param allow_back: indicate if the back command should be proposed.
65
        """
66
        raise NotImplementedError
67

68
    @abstractmethod
2✔
69
    def run_once(self, build_info):
2✔
70
        """
71
        Run the given build and wait for its completion. Return the error
72
        code when available.
73
        """
74
        raise NotImplementedError
75

76
    def index_to_try_after_skip(self, build_range):
2✔
77
        """
78
        Return the index of the build to use after a build was skipped.
79

80
        By default this only returns the mid point of the remaining range.
81
        """
82
        return build_range.mid_point()
×
83

84
    def maybe_snap(self):
2✔
85
        """
86
        Checking if the launcher migth contain Snap specific bits and return
87
        them if it's the case, defaulting to False else.
88
        """
NEW
89
        if hasattr(self, "launcher_kwargs") and "allow_sudo" in self.launcher_kwargs.keys():
×
NEW
90
            return (
×
91
                self.launcher_kwargs["allow_sudo"],
92
                self.launcher_kwargs["disable_snap_connect"],
93
            )
94
        else:
NEW
95
            return (False, False)
×
96

97

98
class ManualTestRunner(TestRunner):
2✔
99
    """
100
    A TestRunner subclass that run builds and ask for evaluation by
101
    prompting in the terminal.
102
    """
103

104
    def __init__(self, launcher_kwargs=None):
2✔
105
        TestRunner.__init__(self)
×
106
        self.launcher_kwargs = launcher_kwargs or {}
×
107

108
    def get_verdict(self, build_info, allow_back):
2✔
109
        """
110
        Ask and returns the verdict.
111
        """
112
        options = ["good", "bad", "skip", "retry", "exit"]
×
113
        if allow_back:
×
114
            options.insert(-1, "back")
×
115
        # allow user to just type one letter
116
        allowed_inputs = options + [o[0] for o in options]
×
117
        # format options to nice printing
118
        formatted_options = ", ".join(["'%s'" % o for o in options[:-1]]) + " or '%s'" % options[-1]
×
119
        verdict = ""
×
120
        while verdict not in allowed_inputs:
×
121
            verdict = input(
×
122
                "Was this %s build good, bad, or broken?"
123
                " (type %s and press Enter): " % (build_info.build_type, formatted_options)
124
            )
125

126
        if verdict == "back":
×
127
            return "back"
×
128
        # shorten verdict to one character for processing...
129
        return verdict[0]
×
130

131
    def evaluate(self, build_info, allow_back=False):
2✔
NEW
132
        with create_launcher(build_info, self.launcher_kwargs) as launcher:
×
133
            launcher.start(**self.launcher_kwargs)
×
134
            build_info.update_from_app_info(launcher.get_app_info())
×
135
            verdict = self.get_verdict(build_info, allow_back)
×
136
            try:
×
137
                launcher.stop()
×
138
            except LauncherError:
×
139
                # we got an error on process termination, but user
140
                # already gave the verdict, so pass this "silently"
141
                # (it would be logged from the launcher anyway)
142
                launcher._running = False
×
143
        return verdict
×
144

145
    def run_once(self, build_info):
2✔
NEW
146
        with create_launcher(
×
147
            build_info, self.launcher_kwargs
148
        ) as launcher:
149
            launcher.start(**self.launcher_kwargs)
×
150
            build_info.update_from_app_info(launcher.get_app_info())
×
151
            return launcher.wait()
×
152

153
    def index_to_try_after_skip(self, build_range):
2✔
154
        mid = TestRunner.index_to_try_after_skip(self, build_range)
×
155
        build_range_len = len(build_range)
×
156
        if build_range_len <= 3:
×
157
            # do not even ask if there is only one build to choose
158
            return mid
×
159
        min = -mid + 1
×
160
        max = build_range_len - mid - 2
×
161
        valid_range = list(range(min, max + 1))
×
162
        print(
×
163
            "Build was skipped. You can manually choose a new build to"
164
            " test, to be able to get out of a broken build range."
165
        )
166
        print(
×
167
            "Please type the index of the build you would like to try - the"
168
            " index is 0-based on the middle of the remaining build range."
169
        )
170
        print("You can choose a build index between [%d, %d]:" % (min, max))
×
171
        while True:
172
            value = input("> ")
×
173
            try:
×
174
                index = int(value)
×
175
                if index in valid_range:
×
176
                    return mid + index
×
177
            except ValueError:
×
178
                pass
179

180

181
def _raise_command_error(exc, msg=""):
2✔
182
    raise TestCommandError("Unable to run the test command%s: `%s`" % (msg, exc))
×
183

184

185
class CommandTestRunner(TestRunner):
2✔
186
    """
187
    A TestRunner subclass that evaluate builds given a shell command.
188

189
    Some variables may be used to evaluate the builds:
190
     - variables referenced in :meth:`TestRunner.evaluate`
191
     - app_name (the tested application name: firefox, ...)
192
     - binary (the path to the binary when applicable - not for fennec)
193

194
    These variables can be used in two ways:
195
    1. as environment variables. 'MOZREGRESSION_' is prepended and the
196
       variables names are upcased. Example: MOZREGRESSION_BINARY
197
    2. as placeholders in the command line. variables names must be enclosed
198
       with curly brackets. Example:
199
       `mozmill -app firefox -b {binary} -t path/to/test.js`
200
    """
201

202
    def __init__(self, command):
2✔
203
        TestRunner.__init__(self)
×
204
        self.command = command
×
205

206
    def evaluate(self, build_info, allow_back=False):
2✔
NEW
207
        with create_launcher(
×
208
            build_info, self.launcher_kwargs
209
        ) as launcher:
210
            build_info.update_from_app_info(launcher.get_app_info())
×
211
            variables = {k: v for k, v in build_info.to_dict().items()}
×
212
            if hasattr(launcher, "binary"):
×
213
                variables["binary"] = launcher.binary
×
214

215
            env = dict(os.environ)
×
216
            for k, v in variables.items():
×
217
                env["MOZREGRESSION_" + k.upper()] = str(v)
×
218
            try:
×
219
                command = self.command.format(**variables)
×
220
            except KeyError as exc:
×
221
                _raise_command_error(exc, " (formatting error)")
×
222
            command = os.path.expanduser(command)
×
223
            LOG.info("Running test command: `%s`" % command)
×
224

225
            # `shlex.split` does parsing and escaping that isn't compatible with Windows.
226
            if sys.platform == "win32":
×
227
                cmdlist = command
×
228
            else:
229
                cmdlist = shlex.split(command)
×
230

231
            try:
×
232
                retcode = subprocess.call(cmdlist, env=env)
×
233
            except IndexError:
×
234
                _raise_command_error("Empty command")
×
235
            except OSError as exc:
×
236
                _raise_command_error(
×
237
                    exc,
238
                    " (%s not found or not executable)"
239
                    % (command if sys.platform == "win32" else cmdlist[0]),
240
                )
241
        LOG.info(
×
242
            "Test command result: %d (build is %s)" % (retcode, "good" if retcode == 0 else "bad")
243
        )
244
        return "g" if retcode == 0 else "b"
×
245

246
    def run_once(self, build_info):
2✔
247
        return 0 if self.evaluate(build_info) == "g" else 1
×
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2025 Coveralls, Inc