• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

mozilla / relman-auto-nag / #4623

pending completion
#4623

push

coveralls-python

web-flow
File potential security crashes as security bugs (#2134)

646 of 3432 branches covered (18.82%)

5 of 5 new or added lines in 2 files covered. (100.0%)

1828 of 8535 relevant lines covered (21.42%)

0.21 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0
/bugbot/crash/analyzer.py
1
# This Source Code Form is subject to the terms of the Mozilla Public
2
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
3
# You can obtain one at http://mozilla.org/MPL/2.0/.
4

5
import itertools
×
6
import re
×
7
from collections import defaultdict
×
8
from datetime import date, timedelta
×
9
from functools import cached_property
×
10
from typing import Iterable, Iterator
×
11

12
from libmozdata import bugzilla, clouseau, connection, socorro
×
13
from libmozdata import utils as lmdutils
×
14
from libmozdata.bugzilla import Bugzilla
×
15
from libmozdata.connection import Connection
×
16

17
from bugbot import logger, utils
×
18
from bugbot.components import ComponentName
×
19
from bugbot.crash import socorro_util
×
20

21
# Allocator poison value addresses.
22
ALLOCATOR_ADDRESSES_64_BIT = (
×
23
    0xE5E5E5E5E5E5E5E5,
24
    0x4B4B4B4B4B4B4B4B,
25
)
26
ALLOCATOR_ADDRESSES_32_BIT = (
×
27
    0xE5E5E5E5,
28
    0x4B4B4B4B,
29
)
30
# The max offset from a memory address to be considered "near".
31
OFFSET_64_BIT = 0x1000
×
32
OFFSET_32_BIT = 0x100
×
33
# Ranges where addresses are considered near allocator poison values.
34
ALLOCATOR_RANGES_64_BIT = (
×
35
    (addr - OFFSET_64_BIT, addr + OFFSET_64_BIT) for addr in ALLOCATOR_ADDRESSES_64_BIT
36
)
37
ALLOCATOR_RANGES_32_BIT = (
×
38
    (addr - OFFSET_32_BIT, addr + OFFSET_32_BIT) for addr in ALLOCATOR_ADDRESSES_32_BIT
39
)
40

41

42
def is_near_null_address(str_address) -> bool:
×
43
    """Check if the address is near null.
44

45
    Args:
46
        str_address: The memory address to check.
47

48
    Returns:
49
        True if the address is near null, False otherwise.
50
    """
51
    address = int(str_address, 0)
×
52
    is_64_bit = len(str_address) >= 18
×
53

54
    if is_64_bit:
×
55
        return -OFFSET_64_BIT <= address <= OFFSET_64_BIT
×
56

57
    return -OFFSET_32_BIT <= address <= OFFSET_32_BIT
×
58

59

60
def is_near_allocator_address(str_address) -> bool:
×
61
    """Check if the address is near an allocator poison value.
62

63
    Args:
64
        str_address: The memory address to check.
65

66
    Returns:
67
        True if the address is near an allocator poison value, False otherwise.
68
    """
69
    address = int(str_address, 0)
×
70
    is_64_bit = len(str_address) >= 18
×
71

72
    return any(
×
73
        low <= address <= high
74
        for low, high in (
75
            ALLOCATOR_RANGES_64_BIT if is_64_bit else ALLOCATOR_RANGES_32_BIT
76
        )
77
    )
78

79

80
# TODO: Move this to libmozdata
81
def generate_signature_page_url(params: dict, tab: str) -> str:
×
82
    """Generate a URL to the signature page on Socorro
83

84
    Args:
85
        params: the parameters for the search query.
86
        tab: the page tab that should be selected.
87

88
    Returns:
89
        The URL of the signature page on Socorro
90
    """
91
    web_url = socorro.Socorro.CRASH_STATS_URL
×
92
    query = lmdutils.get_params_for_url(params)
×
93
    return f"{web_url}/signature/{query}#{tab}"
×
94

95

96
# NOTE: At this point, we will file bugs on bugzilla-dev. Once we are confident
97
# that the bug filing is working as expected, we can switch to filing bugs in
98
# the production instance of Bugzilla.
99
class DevBugzilla(Bugzilla):
×
100
    URL = "https://bugzilla-dev.allizom.org"
×
101
    API_URL = URL + "/rest/bug"
×
102
    ATTACHMENT_API_URL = API_URL + "/attachment"
×
103
    TOKEN = utils.get_login_info()["bz_api_key_dev"]
×
104

105

106
class NoCrashReportFoundError(Exception):
×
107
    """There are no crash reports that meet the required criteria."""
108

109

110
class ClouseauDataAnalyzer:
×
111
    """Analyze the data returned by Crash Clouseau"""
112

113
    MINIMUM_CLOUSEAU_SCORE_THRESHOLD: int = 8
×
114
    DEFAULT_CRASH_COMPONENT = ComponentName("Core", "General")
×
115

116
    def __init__(self, reports: Iterable[dict]):
×
117
        self._clouseau_reports = reports
×
118

119
    @cached_property
×
120
    def max_clouseau_score(self):
×
121
        """The maximum Clouseau score in the crash reports."""
122
        if not self._clouseau_reports:
×
123
            return 0
×
124
        return max(report["max_score"] for report in self._clouseau_reports)
×
125

126
    @cached_property
×
127
    def regressed_by_potential_bug_ids(self) -> set[int]:
×
128
        """The IDs for the bugs that their patches could have caused the crash."""
129
        minimum_accepted_score = max(
×
130
            self.MINIMUM_CLOUSEAU_SCORE_THRESHOLD, self.max_clouseau_score
131
        )
132
        return {
×
133
            changeset["bug_id"]
134
            for report in self._clouseau_reports
135
            if report["max_score"] >= minimum_accepted_score
136
            for changeset in report["changesets"]
137
            if changeset["max_score"] >= minimum_accepted_score
138
            and not changeset["is_merge"]
139
            and not changeset["is_backedout"]
140
        }
141

142
    @cached_property
×
143
    def regressed_by_patch(self) -> str | None:
×
144
        """The hash of the patch that could have caused the crash."""
145
        minimum_accepted_score = max(
×
146
            self.MINIMUM_CLOUSEAU_SCORE_THRESHOLD, self.max_clouseau_score
147
        )
148
        potential_patches = {
×
149
            changeset["changeset"]
150
            for report in self._clouseau_reports
151
            if report["max_score"] >= minimum_accepted_score
152
            for changeset in report["changesets"]
153
            if changeset["max_score"] >= minimum_accepted_score
154
            and not changeset["is_merge"]
155
            and not changeset["is_backedout"]
156
        }
157
        if len(potential_patches) == 1:
×
158
            return next(iter(potential_patches))
×
159
        return None
×
160

161
    @cached_property
×
162
    def regressed_by(self) -> int | None:
×
163
        """The ID of the bug that one of its patches could have caused
164
        the crash.
165

166
        If there are multiple bugs, the value will be `None`.
167
        """
168
        bug_ids = self.regressed_by_potential_bug_ids
×
169
        if len(bug_ids) == 1:
×
170
            return next(iter(bug_ids))
×
171
        return None
×
172

173
    @cached_property
×
174
    def regressed_by_potential_bugs(self) -> list[dict]:
×
175
        """The bugs whose patches could have caused the crash."""
176

177
        def handler(bug: dict, data: list):
×
178
            data.append(bug)
×
179

180
        bugs: list[dict] = []
×
181
        Bugzilla(
×
182
            bugids=self.regressed_by_potential_bug_ids,
183
            include_fields=[
184
                "id",
185
                "groups",
186
                "assigned_to",
187
                "product",
188
                "component",
189
            ],
190
            bughandler=handler,
191
            bugdata=bugs,
192
        ).wait()
193

194
        return bugs
×
195

196
    @cached_property
×
197
    def regressed_by_author(self) -> dict | None:
×
198
        """The author of the patch that could have caused the crash.
199

200
        If there are multiple regressors, the value will be `None`.
201

202
        The regressor bug assignee is considered as the author, even if the
203
        assignee is not the patch author.
204
        """
205

206
        if not self.regressed_by:
×
207
            return None
×
208

209
        bug = self.regressed_by_potential_bugs[0]
×
210
        assert bug["id"] == self.regressed_by
×
211
        return bug["assigned_to_detail"]
×
212

213
    @cached_property
×
214
    def crash_component(self) -> ComponentName:
×
215
        """The component that the crash belongs to.
216

217
        If there are multiple components, the value will be the default one.
218
        """
219
        potential_components = {
×
220
            ComponentName(bug["product"], bug["component"])
221
            for bug in self.regressed_by_potential_bugs
222
        }
223
        if len(potential_components) == 1:
×
224
            return next(iter(potential_components))
×
225
        return self.DEFAULT_CRASH_COMPONENT
×
226

227

228
class SocorroDataAnalyzer(socorro_util.SignatureStats):
×
229
    """Analyze the data returned by Socorro."""
230

231
    _bugzilla_os_legal_values = None
×
232
    _bugzilla_cpu_legal_values_map = None
×
233
    _platforms = [
×
234
        {"short_name": "win", "name": "Windows"},
235
        {"short_name": "mac", "name": "Mac OS X"},
236
        {"short_name": "lin", "name": "Linux"},
237
        {"short_name": "and", "name": "Android"},
238
        {"short_name": "unknown", "name": "Unknown"},
239
    ]
240

241
    def __init__(
×
242
        self,
243
        signature: dict,
244
        num_total_crashes: int,
245
    ):
246
        super().__init__(signature, num_total_crashes, platforms=self._platforms)
×
247

248
    @classmethod
×
249
    def to_bugzilla_op_sys(cls, op_sys: str) -> str:
×
250
        """Return the corresponding OS name in Bugzilla for the provided OS name
251
        from Socorro.
252

253
        If the OS name is not recognized, return "Other".
254
        """
255
        if cls._bugzilla_os_legal_values is None:
×
256
            cls._bugzilla_os_legal_values = set(
×
257
                bugzilla.BugFields.fetch_field_values("op_sys")
258
            )
259

260
        if op_sys in cls._bugzilla_os_legal_values:
×
261
            return op_sys
×
262

263
        if op_sys.startswith("OS X ") or op_sys.startswith("macOS "):
×
264
            op_sys = "macOS"
×
265
        elif op_sys.startswith("Windows"):
×
266
            op_sys = "Windows"
×
267
        elif "Linux" in op_sys or op_sys.startswith("Ubuntu"):
×
268
            op_sys = "Linux"
×
269
        else:
270
            op_sys = "Other"
×
271

272
        return op_sys
×
273

274
    @property
×
275
    def bugzilla_op_sys(self) -> str:
×
276
        """The name of the OS where the crash happens.
277

278
        The value is one of the legal values for Bugzilla's `op_sys` field.
279

280
        - If no OS name is found, the value will be "Unspecified".
281
        - If the OS name is not recognized, the value will be "Other".
282
        - If multiple OS names are found, the value will be "All". Unless the OS
283
          names can be resolved to a common name without a version. For example,
284
          "Windows 10" and "Windows 7" will become "Windows".
285
        """
286
        all_op_sys = {
×
287
            self.to_bugzilla_op_sys(op_sys["term"])
288
            for op_sys in self.signature["facets"]["platform_pretty_version"]
289
        }
290

291
        if len(all_op_sys) > 1:
×
292
            # Resolve to root OS name by removing the version number.
293
            all_op_sys = {op_sys.split(" ")[0] for op_sys in all_op_sys}
×
294

295
        if len(all_op_sys) == 2 and "Other" in all_op_sys:
×
296
            # TODO: explain this workaround.
297
            all_op_sys.remove("Other")
×
298

299
        if len(all_op_sys) == 1:
×
300
            return next(iter(all_op_sys))
×
301

302
        if len(all_op_sys) == 0:
×
303
            return "Unspecified"
×
304

305
        return "All"
×
306

307
    @classmethod
×
308
    def to_bugzilla_cpu(cls, cpu: str) -> str:
×
309
        """Return the corresponding CPU name in Bugzilla for the provided name
310
        from Socorro.
311

312
        If the CPU is not recognized, return "Other".
313
        """
314
        if cls._bugzilla_cpu_legal_values_map is None:
×
315
            cls._bugzilla_cpu_legal_values_map = {
×
316
                value.lower(): value
317
                for value in bugzilla.BugFields.fetch_field_values("rep_platform")
318
            }
319

320
        return cls._bugzilla_cpu_legal_values_map.get(cpu, "Other")
×
321

322
    @property
×
323
    def bugzilla_cpu_arch(self) -> str:
×
324
        """The CPU architecture of the devices where the crash happens.
325

326
        The value is one of the legal values for Bugzilla's `rep_platform` field.
327

328
        - If no CPU architecture is found, the value will be "Unspecified".
329
        - If the CPU architecture is not recognized, the value will be "Other".
330
        - If multiple CPU architectures are found, the value will "All".
331
        """
332
        all_cpu_arch = {
×
333
            self.to_bugzilla_cpu(cpu["term"])
334
            for cpu in self.signature["facets"]["cpu_arch"]
335
        }
336

337
        if len(all_cpu_arch) == 2 and "Other" in all_cpu_arch:
×
338
            all_cpu_arch.remove("Other")
×
339

340
        if len(all_cpu_arch) == 1:
×
341
            return next(iter(all_cpu_arch))
×
342

343
        if len(all_cpu_arch) == 0:
×
344
            return "Unspecified"
×
345

346
        return "All"
×
347

348
    @property
×
349
    def user_comments_page_url(self) -> str:
×
350
        """The URL to the Signature page on Socorro where the Comments tab is
351
        selected.
352
        """
353
        start_date = date.today() - timedelta(weeks=26)
×
354
        params = {
×
355
            "signature": self.signature_term,
356
            "date": socorro.SuperSearch.get_search_date(start_date),
357
        }
358
        return generate_signature_page_url(params, "comments")
×
359

360
    @property
×
361
    def num_user_comments(self) -> int:
×
362
        """The number of crash reports with user comments."""
363
        # TODO: count useful/interesting user comments (e.g., exclude one word comments)
364
        return self.signature["facets"]["cardinality_user_comments"]["value"]
×
365

366
    @property
×
367
    def has_user_comments(self) -> bool:
×
368
        """Whether the crash signature has any reports with a user comment."""
369
        return self.num_user_comments > 0
×
370

371
    @property
×
372
    def top_proto_signature(self) -> str:
×
373
        """The proto signature that occurs the most."""
374
        return self.signature["facets"]["proto_signature"][0]["term"]
×
375

376
    @property
×
377
    def num_top_proto_signature_crashes(self) -> int:
×
378
        """The number of crashes for the most occurring proto signature."""
379
        return self.signature["facets"]["proto_signature"][0]["count"]
×
380

381
    def _build_ids(self) -> Iterator[int]:
×
382
        """Yields the build IDs where the crash occurred."""
383
        for build_id in self.signature["facets"]["build_id"]:
×
384
            yield build_id["term"]
×
385

386
    @property
×
387
    def top_build_id(self) -> int:
×
388
        """The build ID where most crashes occurred."""
389
        return self.signature["facets"]["build_id"][0]["term"]
×
390

391
    @cached_property
×
392
    def num_near_null_crashes(self) -> int:
×
393
        """The number of crashes that occurred on addresses near null."""
394
        return sum(
×
395
            address["count"]
396
            for address in self.signature["facets"]["address"]
397
            if is_near_null_address(address["term"])
398
        )
399

400
    @property
×
401
    def is_near_null_crash(self) -> bool:
×
402
        """Whether all crashes occurred on addresses near null."""
403
        return self.num_near_null_crashes == self.num_crashes
×
404

405
    @property
×
406
    def is_potential_near_null_crash(self) -> bool:
×
407
        """Whether the signature is a potential near null crash.
408

409
        The value will be True if some but not all crashes occurred on addresses
410
        near null.
411
        """
412
        return not self.is_near_null_crash and self.num_near_null_crashes > 0
×
413

414
    @property
×
415
    def is_near_null_related_crash(self) -> bool:
×
416
        """Whether the signature is related to near null crashes.
417

418
        The value will be True if any of the crashes occurred on addresses near
419
        null.
420
        """
421
        return self.is_near_null_crash or self.is_potential_near_null_crash
×
422

423
    @cached_property
×
424
    def num_near_allocator_crashes(self) -> int:
×
425
        """The number of crashes that occurred on addresses near an allocator
426
        poison value.
427
        """
428
        return sum(
×
429
            address["count"]
430
            for address in self.signature["facets"]["address"]
431
            if is_near_allocator_address(address["term"])
432
        )
433

434
    @property
×
435
    def is_near_allocator_crash(self) -> bool:
×
436
        """Whether all crashes occurred on addresses near an allocator poison
437
        value.
438
        """
439
        return self.num_near_allocator_crashes == self.num_crashes
×
440

441
    @property
×
442
    def is_potential_near_allocator_crash(self) -> bool:
×
443
        """Whether the signature is a potential near allocator poison value
444
        crash.
445

446
        The value will be True if some but not all crashes occurred on addresses
447
        near an allocator poison value.
448
        """
449
        return not self.is_near_allocator_crash and self.num_near_allocator_crashes > 0
×
450

451
    @property
×
452
    def is_near_allocator_related_crash(self) -> bool:
×
453
        """Whether the signature is related to near allocator poison value
454
        crashes.
455

456
        The value will be True if any of the crashes occurred on addresses near
457
        an allocator poison value.
458
        """
459
        return self.is_near_allocator_crash or self.is_potential_near_allocator_crash
×
460

461

462
class SignatureAnalyzer(SocorroDataAnalyzer, ClouseauDataAnalyzer):
×
463
    """Analyze the data related to a signature.
464

465
    This includes data from Socorro and Clouseau.
466
    """
467

468
    def __init__(
×
469
        self,
470
        socorro_signature: dict,
471
        num_total_crashes: int,
472
        clouseau_reports: list[dict],
473
    ):
474
        SocorroDataAnalyzer.__init__(self, socorro_signature, num_total_crashes)
×
475
        ClouseauDataAnalyzer.__init__(self, clouseau_reports)
×
476

477
    def _fetch_crash_reports(
×
478
        self,
479
        proto_signature: str,
480
        build_id: int | Iterable[int],
481
        limit: int = 1,
482
    ) -> Iterator[dict]:
483
        params = {
×
484
            "proto_signature": "=" + proto_signature,
485
            "build_id": build_id,
486
            "_columns": [
487
                "uuid",
488
            ],
489
            "_results_number": limit,
490
        }
491

492
        def handler(res: dict, data: dict):
×
493
            data.update(res)
×
494

495
        data: dict = {}
×
496
        socorro.SuperSearch(params=params, handler=handler, handlerdata=data).wait()
×
497

498
        yield from data["hits"]
×
499

500
    def fetch_representative_processed_crash(self) -> dict:
×
501
        """Fetch a processed crash to represent the signature.
502

503
        This could fetch multiple processed crashes and return the one that is
504
        most likely to be useful.
505
        """
506
        limit_to_top_proto_signature = (
×
507
            self.num_top_proto_signature_crashes / self.num_crashes > 0.6
508
        )
509

510
        reports = itertools.chain(
×
511
            # Reports with a higher score from clouseau are more likely to be
512
            # useful.
513
            sorted(
514
                self._clouseau_reports,
515
                key=lambda report: report["max_score"],
516
                reverse=True,
517
            ),
518
            # Next we try find reports from the top crashing build because they
519
            # are likely to be representative.
520
            self._fetch_crash_reports(self.top_proto_signature, self.top_build_id),
521
            self._fetch_crash_reports(self.top_proto_signature, self._build_ids()),
522
        )
523
        for report in reports:
×
524
            uuid = report["uuid"]
×
525
            processed_crash = socorro.ProcessedCrash.get_processed(uuid)[uuid]
×
526
            if (
×
527
                not limit_to_top_proto_signature
528
                or processed_crash["proto_signature"] == self.top_proto_signature
529
            ):
530
                # TODO(investigate): maybe we should check if the stack is
531
                # corrupted (ask gsvelto or willkg about how to detect that)
532
                return processed_crash
×
533

534
        raise NoCrashReportFoundError(
×
535
            f"No crash report found with the most frequent proto signature for {self.signature_term}."
536
        )
537

538
    @cached_property
×
539
    def is_potential_security_crash(self) -> bool:
×
540
        """Whether the crash is related to a potential security bug.
541

542
        The value will be True if:
543
            - the signature is related to near allocator poison value crashes, or
544
            - one of the potential regressors is a security bug
545
        """
546
        return self.is_near_allocator_related_crash or any(
×
547
            any("core-security" in group for group in bug["groups"])
548
            for bug in self.regressed_by_potential_bugs
549
        )
550

551

552
class SignaturesDataFetcher:
×
553
    """Fetch the data related to the given signatures."""
554

555
    MEMORY_ACCESS_ERROR_REASONS = (
×
556
        # On Windows:
557
        "EXCEPTION_ACCESS_VIOLATION_READ",
558
        "EXCEPTION_ACCESS_VIOLATION_WRITE",
559
        "EXCEPTION_ACCESS_VIOLATION_EXEC"
560
        # On Linux:
561
        "SIGSEGV / SEGV_MAPERR",
562
        "SIGSEGV / SEGV_ACCERR",
563
    )
564

565
    EXCLUDED_MOZ_REASON_STRINGS = (
×
566
        "MOZ_CRASH(OOM)",
567
        "MOZ_CRASH(Out of memory)",
568
        "out of memory",
569
        "Shutdown hanging",
570
        # TODO(investigate): do we need to exclude signatures that their reason
571
        # contains `[unhandlable oom]`?
572
        # Example: arena_t::InitChunk | arena_t::AllocRun | arena_t::MallocLarge | arena_t::Malloc | BaseAllocator::malloc | Allocator::malloc | PageMalloc
573
        # "[unhandlable oom]",
574
    )
575

576
    # If any of the crash reason starts with any of the following, then it is
577
    # Network or I/O error.
578
    EXCLUDED_IO_ERROR_REASON_PREFIXES = (
×
579
        "EXCEPTION_IN_PAGE_ERROR_READ",
580
        "EXCEPTION_IN_PAGE_ERROR_WRITE",
581
        "EXCEPTION_IN_PAGE_ERROR_EXEC",
582
    )
583

584
    # TODO(investigate): do we need to exclude all these signatures prefixes?
585
    EXCLUDED_SIGNATURE_PREFIXES = (
×
586
        "OOM | ",
587
        "bad hardware | ",
588
        "shutdownhang | ",
589
    )
590

591
    def __init__(
×
592
        self,
593
        signatures: Iterable[str],
594
        product: str = "Firefox",
595
        channel: str = "nightly",
596
    ):
597
        self._signatures = set(signatures)
×
598
        self._product = product
×
599
        self._channel = channel
×
600

601
    @classmethod
×
602
    def find_new_actionable_crashes(
×
603
        cls,
604
        product: str,
605
        channel: str,
606
        days_to_check: int = 7,
607
        days_without_crashes: int = 7,
608
    ) -> "SignaturesDataFetcher":
609
        """Find new actionable crashes.
610

611
        Args:
612
            product: The product to check.
613
            channel: The release channel to check.
614
            days_to_check: The number of days to check for crashes.
615
            days_without_crashes: The number of days without crashes before the
616
                `days_to_check` to consider the signature new.
617

618
        Returns:
619
            A list of actionable signatures.
620
        """
621
        duration = days_to_check + days_without_crashes
×
622
        end_date = lmdutils.get_date_ymd("today")
×
623
        start_date = end_date - timedelta(duration)
×
624
        earliest_allowed_date = lmdutils.get_date_str(
×
625
            end_date - timedelta(days_to_check)
626
        )
627
        date_range = socorro.SuperSearch.get_search_date(start_date, end_date)
×
628

629
        params = {
×
630
            "product": product,
631
            "release_channel": channel,
632
            "date": date_range,
633
            # TODO(investigate): should we do a local filter instead of the
634
            # following (should we exclude the signature if one of the crashes
635
            # is a shutdown hang?):
636
            # If the `ipc_shutdown_state` or `shutdown_progress` field are
637
            # non-empty then it's a shutdown hang.
638
            "ipc_shutdown_state": "__null__",
639
            "shutdown_progress": "__null__",
640
            # TODO(investigate): should we use the following instead of the
641
            # local filter.
642
            # "oom_allocation_size": "!__null__",
643
            "_aggs.signature": [
644
                "moz_crash_reason",
645
                "reason",
646
                "_histogram.date",
647
                "_cardinality.install_time",
648
                "_cardinality.oom_allocation_size",
649
            ],
650
            "_results_number": 0,
651
            "_facets_size": 10000,
652
        }
653

654
        def handler(search_resp: dict, data: list):
×
655
            logger.debug(
×
656
                "Total of %d signatures received from Socorro",
657
                len(search_resp["facets"]["signature"]),
658
            )
659

660
            for crash in search_resp["facets"]["signature"]:
×
661
                signature = crash["term"]
×
662
                if any(
×
663
                    signature.startswith(excluded_prefix)
664
                    for excluded_prefix in cls.EXCLUDED_SIGNATURE_PREFIXES
665
                ):
666
                    # Ignore signatures that start with any of the excluded prefixes.
667
                    continue
×
668

669
                facets = crash["facets"]
×
670
                installations = facets["cardinality_install_time"]["value"]
×
671
                if installations <= 1:
×
672
                    # Ignore crashes that only happen on one installation.
673
                    continue
×
674

675
                first_date = facets["histogram_date"][0]["term"]
×
676
                if first_date < earliest_allowed_date:
×
677
                    # The crash is not new, skip it.
678
                    continue
×
679

680
                if any(
×
681
                    reason["term"].startswith(io_error_prefix)
682
                    for reason in facets["reason"]
683
                    for io_error_prefix in cls.EXCLUDED_IO_ERROR_REASON_PREFIXES
684
                ):
685
                    # Ignore Network or I/O error crashes.
686
                    continue
×
687

688
                if crash["count"] < 20:
×
689
                    # For signatures with low volume, having multiple types of
690
                    # memory errors indicates potential bad hardware crashes.
691
                    num_memory_error_types = sum(
×
692
                        reason["term"] in cls.MEMORY_ACCESS_ERROR_REASONS
693
                        for reason in facets["reason"]
694
                    )
695
                    if num_memory_error_types > 1:
×
696
                        # Potential bad hardware crash, skip it.
697
                        continue
×
698

699
                # TODO: Add a filter using the `possible_bit_flips_max_confidence`
700
                # field to exclude bad hardware crashes. The filed is not available yet.
701
                # See: https://bugzilla.mozilla.org/show_bug.cgi?id=1816669#c3
702

703
                # TODO(investigate): is this needed since we are already
704
                # filtering signatures that start with "OOM | "
705
                if facets["cardinality_oom_allocation_size"]["value"]:
×
706
                    # If one of the crashes is an OOM crash, skip it.
707
                    continue
×
708

709
                # TODO(investigate): do we need to check for the `moz_crash_reason`
710
                moz_crash_reasons = facets["moz_crash_reason"]
×
711
                if moz_crash_reasons and any(
×
712
                    excluded_reason in reason["term"]
713
                    for reason in moz_crash_reasons
714
                    for excluded_reason in cls.EXCLUDED_MOZ_REASON_STRINGS
715
                ):
716
                    continue
×
717

718
                data.append(signature)
×
719

720
        signatures: list = []
×
721
        socorro.SuperSearch(
×
722
            params=params,
723
            handler=handler,
724
            handlerdata=signatures,
725
        ).wait()
726

727
        logger.debug(
×
728
            "Total of %d signatures left after applying the filtering criteria",
729
            len(signatures),
730
        )
731

732
        return cls(signatures, product, channel)
×
733

734
    def fetch_clouseau_crash_reports(self) -> dict[str, list]:
×
735
        """Fetch the crash reports data from Crash Clouseau."""
736
        signature_reports = clouseau.Reports.get_by_signatures(
×
737
            self._signatures,
738
            product=self._product,
739
            channel=self._channel,
740
        )
741

742
        logger.debug(
×
743
            "Total of %d signatures received from Clouseau", len(signature_reports)
744
        )
745

746
        return signature_reports
×
747

748
    def fetch_socorro_info(self) -> tuple[list[dict], int]:
×
749
        """Fetch the signature data from Socorro."""
750
        # TODO(investigate): should we increase the duration to 6 months?
751
        duration = timedelta(weeks=1)
×
752
        end_date = lmdutils.get_date_ymd("today")
×
753
        start_date = end_date - duration
×
754
        date_range = socorro.SuperSearch.get_search_date(start_date, end_date)
×
755

756
        params = {
×
757
            "product": self._product,
758
            # TODO(investigate): should we included all release channels?
759
            "release_channel": self._channel,
760
            # TODO(investigate): should we limit based on the build date as well?
761
            "date": date_range,
762
            # TODO: split signatures into chunks to avoid very long query URLs
763
            "signature": ["=" + signature for signature in self._signatures],
764
            "_aggs.signature": [
765
                "address",
766
                "build_id",
767
                "cpu_arch",
768
                "proto_signature",
769
                "_cardinality.user_comments",
770
                "cpu_arch",
771
                "platform_pretty_version",
772
                # The following are needed for SignatureStats:
773
                "platform",
774
                "is_garbage_collecting",
775
                "_cardinality.install_time",
776
                "startup_crash",
777
                "_histogram.uptime",
778
                "process_type",
779
            ],
780
            "_results_number": 0,
781
            "_facets_size": 10000,
782
        }
783

784
        def handler(search_results: dict, data: dict):
×
785
            data["num_total_crashes"] = search_results["total"]
×
786
            data["signatures"] = search_results["facets"]["signature"]
×
787

788
        data: dict = {}
×
789
        socorro.SuperSearchUnredacted(
×
790
            params=params,
791
            handler=handler,
792
            handlerdata=data,
793
        ).wait()
794

795
        logger.debug(
×
796
            "Fetch info from Socorro for %d signatures", len(data["signatures"])
797
        )
798

799
        return data["signatures"], data["num_total_crashes"]
×
800

801
    def fetch_bugs(self, include_fields: list[str] = None) -> dict[str, list[dict]]:
×
802
        """Fetch bugs that are filed against the given signatures."""
803

804
        params_base: dict = {
×
805
            "include_fields": [
806
                "cf_crash_signature",
807
            ],
808
        }
809

810
        if include_fields:
×
811
            params_base["include_fields"].extend(include_fields)
×
812

813
        params_list = []
×
814
        for signatures_chunk in Connection.chunks(list(self._signatures), 30):
×
815
            params = params_base.copy()
×
816
            n = int(utils.get_last_field_num(params))
×
817
            params[f"f{n}"] = "OP"
×
818
            params[f"j{n}"] = "OR"
×
819
            for signature in signatures_chunk:
×
820
                n += 1
×
821
                params[f"f{n}"] = "cf_crash_signature"
×
822
                params[f"o{n}"] = "regexp"
×
823
                params[f"v{n}"] = rf"\[(@ |@){re.escape(signature)}( \]|\])"
×
824
            params[f"f{n+1}"] = "CP"
×
825
            params_list.append(params)
×
826

827
        signatures_bugs: dict = defaultdict(list)
×
828

829
        def handler(res, data):
×
830
            for bug in res["bugs"]:
×
831
                for signature in utils.get_signatures(bug["cf_crash_signature"]):
×
832
                    if signature in self._signatures:
×
833
                        data[signature].append(bug)
×
834

835
        Bugzilla(
×
836
            queries=[
837
                connection.Query(Bugzilla.API_URL, params, handler, signatures_bugs)
838
                for params in params_list
839
            ],
840
        ).wait()
841

842
        # TODO: remove the call to DevBugzilla after moving to production
843
        DevBugzilla(
×
844
            queries=[
845
                connection.Query(DevBugzilla.API_URL, params, handler, signatures_bugs)
846
                for params in params_list
847
            ],
848
        ).wait()
849

850
        logger.debug(
×
851
            "Total of %d signatures already have bugs filed", len(signatures_bugs)
852
        )
853

854
        return signatures_bugs
×
855

856
    def analyze(self) -> list[SignatureAnalyzer]:
×
857
        """Analyze the data related to the signatures."""
858
        bugs = self.fetch_bugs()
×
859
        # TODO(investigate): For now, we are ignoring signatures that have bugs
860
        # filed even if they are closed long time ago. We should investigate
861
        # whether we should include the ones with closed bugs. For example, if
862
        # the bug was closed as Fixed years ago.
863
        self._signatures.difference_update(bugs.keys())
×
864

865
        clouseau_reports = self.fetch_clouseau_crash_reports()
×
866
        # TODO(investigate): For now, we are ignoring signatures that are not
867
        # analyzed by clouseau. We should investigate why they are not analyzed
868
        # and whether we should include them.
869
        self._signatures.intersection_update(clouseau_reports.keys())
×
870

871
        signatures, num_total_crashes = self.fetch_socorro_info()
×
872
        logger.debug("Total of %d signatures will be analyzed", len(signatures))
×
873

874
        return [
×
875
            SignatureAnalyzer(
876
                signature,
877
                num_total_crashes,
878
                clouseau_reports[signature["term"]],
879
            )
880
            for signature in signatures
881
        ]
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2025 Coveralls, Inc