• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

mozilla / relman-auto-nag / #4616

pending completion
#4616

push

coveralls-python

suhaibmujahid
Highlight crash address commonalities

646 of 3428 branches covered (18.84%)

40 of 40 new or added lines in 1 file covered. (100.0%)

1828 of 8530 relevant lines covered (21.43%)

0.21 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0
/bugbot/crash/analyzer.py
1
# This Source Code Form is subject to the terms of the Mozilla Public
2
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
3
# You can obtain one at http://mozilla.org/MPL/2.0/.
4

5
import itertools
×
6
import re
×
7
from collections import defaultdict
×
8
from datetime import date, timedelta
×
9
from functools import cached_property
×
10
from typing import Iterable, Iterator
×
11

12
from libmozdata import bugzilla, clouseau, connection, socorro
×
13
from libmozdata import utils as lmdutils
×
14
from libmozdata.bugzilla import Bugzilla
×
15
from libmozdata.connection import Connection
×
16

17
from bugbot import logger, utils
×
18
from bugbot.components import ComponentName
×
19
from bugbot.crash import socorro_util
×
20

21
# Addresses of the allocator, which is used to determine if a crash is near the
22
# allocator position.
23
ALLOCATOR_ADDRESSES_64_BIT = (
×
24
    0xE5E5E5E5E5E5E5E5,
25
    0x4B4B4B4B4B4B4B4B,
26
)
27
ALLOCATOR_ADDRESSES_32_BIT = (
×
28
    0xE5E5E5E5,
29
    0x4B4B4B4B,
30
)
31
# Offsets from the allocator address that are considered "near" the allocator
32
# position.
33
OFFSET_64_BIT = 0x1000
×
34
OFFSET_32_BIT = 0x100
×
35
# Ranges of addresses that are considered "near" the allocator position based on
36
# the allocator address and the offset.
37
ALLOCATOR_RANGES_64_BIT = (
×
38
    (addr - OFFSET_64_BIT, addr + OFFSET_64_BIT) for addr in ALLOCATOR_ADDRESSES_64_BIT
39
)
40
ALLOCATOR_RANGES_32_BIT = (
×
41
    (addr - OFFSET_32_BIT, addr + OFFSET_32_BIT) for addr in ALLOCATOR_ADDRESSES_32_BIT
42
)
43

44

45
def is_near_null_address(str_address) -> bool:
×
46
    """Check if the address is near the null address.
47

48
    Args:
49
        str_address: The memory address to check.
50

51
    Returns:
52
        True if the address is near the null address, False otherwise.
53
    """
54
    address = int(str_address, 0)
×
55
    is_64_bit = len(str_address) >= 18
×
56

57
    if is_64_bit:
×
58
        return -OFFSET_64_BIT <= address <= OFFSET_64_BIT
×
59

60
    return -OFFSET_32_BIT <= address <= OFFSET_32_BIT
×
61

62

63
def is_near_allocator_address(str_address) -> bool:
×
64
    """Check if the address is near the allocator address.
65

66
    Args:
67
        str_address: The memory address to check.
68

69
    Returns:
70
        True if the address is near the allocator address, False otherwise.
71
    """
72
    address = int(str_address, 0)
×
73
    is_64_bit = len(str_address) >= 18
×
74

75
    return any(
×
76
        low <= address <= high
77
        for low, high in (
78
            ALLOCATOR_RANGES_64_BIT if is_64_bit else ALLOCATOR_RANGES_32_BIT
79
        )
80
    )
81

82

83
# TODO: Move this to libmozdata
84
def generate_signature_page_url(params: dict, tab: str) -> str:
×
85
    """Generate a URL to the signature page on Socorro
86

87
    Args:
88
        params: the parameters for the search query.
89
        tab: the page tab that should be selected.
90

91
    Returns:
92
        The URL of the signature page on Socorro
93
    """
94
    web_url = socorro.Socorro.CRASH_STATS_URL
×
95
    query = lmdutils.get_params_for_url(params)
×
96
    return f"{web_url}/signature/{query}#{tab}"
×
97

98

99
# NOTE: At this point, we will file bugs on bugzilla-dev. Once we are confident
100
# that the bug filing is working as expected, we can switch to filing bugs in
101
# the production instance of Bugzilla.
102
class DevBugzilla(Bugzilla):
×
103
    URL = "https://bugzilla-dev.allizom.org"
×
104
    API_URL = URL + "/rest/bug"
×
105
    ATTACHMENT_API_URL = API_URL + "/attachment"
×
106
    TOKEN = utils.get_login_info()["bz_api_key_dev"]
×
107

108

109
class NoCrashReportFoundError(Exception):
×
110
    """There are no crash reports that meet the required criteria."""
111

112

113
class ClouseauDataAnalyzer:
×
114
    """Analyze the data returned by Crash Clouseau"""
115

116
    MINIMUM_CLOUSEAU_SCORE_THRESHOLD: int = 8
×
117
    DEFAULT_CRASH_COMPONENT = ComponentName("Core", "General")
×
118

119
    def __init__(self, reports: Iterable[dict]):
×
120
        self._clouseau_reports = reports
×
121

122
    @cached_property
×
123
    def max_clouseau_score(self):
×
124
        """The maximum Clouseau score in the crash reports."""
125
        if not self._clouseau_reports:
×
126
            return 0
×
127
        return max(report["max_score"] for report in self._clouseau_reports)
×
128

129
    @cached_property
×
130
    def regressed_by_potential_bug_ids(self) -> set[int]:
×
131
        """The IDs for the bugs that their patches could have caused the crash."""
132
        minimum_accepted_score = max(
×
133
            self.MINIMUM_CLOUSEAU_SCORE_THRESHOLD, self.max_clouseau_score
134
        )
135
        return {
×
136
            changeset["bug_id"]
137
            for report in self._clouseau_reports
138
            if report["max_score"] >= minimum_accepted_score
139
            for changeset in report["changesets"]
140
            if changeset["max_score"] >= minimum_accepted_score
141
            and not changeset["is_merge"]
142
            and not changeset["is_backedout"]
143
        }
144

145
    @cached_property
×
146
    def regressed_by_patch(self) -> str | None:
×
147
        """The hash of the patch that could have caused the crash."""
148
        minimum_accepted_score = max(
×
149
            self.MINIMUM_CLOUSEAU_SCORE_THRESHOLD, self.max_clouseau_score
150
        )
151
        potential_patches = {
×
152
            changeset["changeset"]
153
            for report in self._clouseau_reports
154
            if report["max_score"] >= minimum_accepted_score
155
            for changeset in report["changesets"]
156
            if changeset["max_score"] >= minimum_accepted_score
157
            and not changeset["is_merge"]
158
            and not changeset["is_backedout"]
159
        }
160
        if len(potential_patches) == 1:
×
161
            return next(iter(potential_patches))
×
162
        return None
×
163

164
    @cached_property
×
165
    def regressed_by(self) -> int | None:
×
166
        """The ID of the bug that one of its patches could have caused
167
        the crash.
168

169
        If there are multiple bugs, the value will be `None`.
170
        """
171
        bug_ids = self.regressed_by_potential_bug_ids
×
172
        if len(bug_ids) == 1:
×
173
            return next(iter(bug_ids))
×
174
        return None
×
175

176
    @cached_property
×
177
    def regressed_by_potential_bugs(self) -> list[dict]:
×
178
        """The bugs whose patches could have caused the crash."""
179

180
        def handler(bug: dict, data: list):
×
181
            data.append(bug)
×
182

183
        bugs: list[dict] = []
×
184
        Bugzilla(
×
185
            bugids=self.regressed_by_potential_bug_ids,
186
            include_fields=[
187
                "id",
188
                "assigned_to",
189
                "product",
190
                "component",
191
            ],
192
            bughandler=handler,
193
            bugdata=bugs,
194
        ).wait()
195

196
        return bugs
×
197

198
    @cached_property
×
199
    def regressed_by_author(self) -> dict | None:
×
200
        """The author of the patch that could have caused the crash.
201

202
        If there are multiple regressors, the value will be `None`.
203

204
        The regressor bug assignee is considered as the author, even if the
205
        assignee is not the patch author.
206
        """
207

208
        if not self.regressed_by:
×
209
            return None
×
210

211
        bug = self.regressed_by_potential_bugs[0]
×
212
        assert bug["id"] == self.regressed_by
×
213
        return bug["assigned_to_detail"]
×
214

215
    @cached_property
×
216
    def crash_component(self) -> ComponentName:
×
217
        """The component that the crash belongs to.
218

219
        If there are multiple components, the value will be the default one.
220
        """
221
        potential_components = {
×
222
            ComponentName(bug["product"], bug["component"])
223
            for bug in self.regressed_by_potential_bugs
224
        }
225
        if len(potential_components) == 1:
×
226
            return next(iter(potential_components))
×
227
        return self.DEFAULT_CRASH_COMPONENT
×
228

229

230
class SocorroDataAnalyzer(socorro_util.SignatureStats):
×
231
    """Analyze the data returned by Socorro."""
232

233
    _bugzilla_os_legal_values = None
×
234
    _bugzilla_cpu_legal_values_map = None
×
235
    _platforms = [
×
236
        {"short_name": "win", "name": "Windows"},
237
        {"short_name": "mac", "name": "Mac OS X"},
238
        {"short_name": "lin", "name": "Linux"},
239
        {"short_name": "and", "name": "Android"},
240
        {"short_name": "unknown", "name": "Unknown"},
241
    ]
242

243
    def __init__(
×
244
        self,
245
        signature: dict,
246
        num_total_crashes: int,
247
    ):
248
        super().__init__(signature, num_total_crashes, platforms=self._platforms)
×
249

250
    @classmethod
×
251
    def to_bugzilla_op_sys(cls, op_sys: str) -> str:
×
252
        """Return the corresponding OS name in Bugzilla for the provided OS name
253
        from Socorro.
254

255
        If the OS name is not recognized, return "Other".
256
        """
257
        if cls._bugzilla_os_legal_values is None:
×
258
            cls._bugzilla_os_legal_values = set(
×
259
                bugzilla.BugFields.fetch_field_values("op_sys")
260
            )
261

262
        if op_sys in cls._bugzilla_os_legal_values:
×
263
            return op_sys
×
264

265
        if op_sys.startswith("OS X ") or op_sys.startswith("macOS "):
×
266
            op_sys = "macOS"
×
267
        elif op_sys.startswith("Windows"):
×
268
            op_sys = "Windows"
×
269
        elif "Linux" in op_sys or op_sys.startswith("Ubuntu"):
×
270
            op_sys = "Linux"
×
271
        else:
272
            op_sys = "Other"
×
273

274
        return op_sys
×
275

276
    @property
×
277
    def bugzilla_op_sys(self) -> str:
×
278
        """The name of the OS where the crash happens.
279

280
        The value is one of the legal values for Bugzilla's `op_sys` field.
281

282
        - If no OS name is found, the value will be "Unspecified".
283
        - If the OS name is not recognized, the value will be "Other".
284
        - If multiple OS names are found, the value will be "All". Unless the OS
285
          names can be resolved to a common name without a version. For example,
286
          "Windows 10" and "Windows 7" will become "Windows".
287
        """
288
        all_op_sys = {
×
289
            self.to_bugzilla_op_sys(op_sys["term"])
290
            for op_sys in self.signature["facets"]["platform_pretty_version"]
291
        }
292

293
        if len(all_op_sys) > 1:
×
294
            # Resolve to root OS name by removing the version number.
295
            all_op_sys = {op_sys.split(" ")[0] for op_sys in all_op_sys}
×
296

297
        if len(all_op_sys) == 2 and "Other" in all_op_sys:
×
298
            # TODO: explain this workaround.
299
            all_op_sys.remove("Other")
×
300

301
        if len(all_op_sys) == 1:
×
302
            return next(iter(all_op_sys))
×
303

304
        if len(all_op_sys) == 0:
×
305
            return "Unspecified"
×
306

307
        return "All"
×
308

309
    @classmethod
×
310
    def to_bugzilla_cpu(cls, cpu: str) -> str:
×
311
        """Return the corresponding CPU name in Bugzilla for the provided name
312
        from Socorro.
313

314
        If the CPU is not recognized, return "Other".
315
        """
316
        if cls._bugzilla_cpu_legal_values_map is None:
×
317
            cls._bugzilla_cpu_legal_values_map = {
×
318
                value.lower(): value
319
                for value in bugzilla.BugFields.fetch_field_values("rep_platform")
320
            }
321

322
        return cls._bugzilla_cpu_legal_values_map.get(cpu, "Other")
×
323

324
    @property
×
325
    def bugzilla_cpu_arch(self) -> str:
×
326
        """The CPU architecture of the devices where the crash happens.
327

328
        The value is one of the legal values for Bugzilla's `rep_platform` field.
329

330
        - If no CPU architecture is found, the value will be "Unspecified".
331
        - If the CPU architecture is not recognized, the value will be "Other".
332
        - If multiple CPU architectures are found, the value will "All".
333
        """
334
        all_cpu_arch = {
×
335
            self.to_bugzilla_cpu(cpu["term"])
336
            for cpu in self.signature["facets"]["cpu_arch"]
337
        }
338

339
        if len(all_cpu_arch) == 2 and "Other" in all_cpu_arch:
×
340
            all_cpu_arch.remove("Other")
×
341

342
        if len(all_cpu_arch) == 1:
×
343
            return next(iter(all_cpu_arch))
×
344

345
        if len(all_cpu_arch) == 0:
×
346
            return "Unspecified"
×
347

348
        return "All"
×
349

350
    @property
×
351
    def user_comments_page_url(self) -> str:
×
352
        """The URL to the Signature page on Socorro where the Comments tab is
353
        selected.
354
        """
355
        start_date = date.today() - timedelta(weeks=26)
×
356
        params = {
×
357
            "signature": self.signature_term,
358
            "date": socorro.SuperSearch.get_search_date(start_date),
359
        }
360
        return generate_signature_page_url(params, "comments")
×
361

362
    @property
×
363
    def num_user_comments(self) -> int:
×
364
        """The number of crash reports with user comments."""
365
        # TODO: count useful/interesting user comments (e.g., exclude one word comments)
366
        return self.signature["facets"]["cardinality_user_comments"]["value"]
×
367

368
    @property
×
369
    def has_user_comments(self) -> bool:
×
370
        """Whether the crash signature has any reports with a user comment."""
371
        return self.num_user_comments > 0
×
372

373
    @property
×
374
    def top_proto_signature(self) -> str:
×
375
        """The proto signature that occurs the most."""
376
        return self.signature["facets"]["proto_signature"][0]["term"]
×
377

378
    @property
×
379
    def num_top_proto_signature_crashes(self) -> int:
×
380
        """The number of crashes for the most occurring proto signature."""
381
        return self.signature["facets"]["proto_signature"][0]["count"]
×
382

383
    def _build_ids(self) -> Iterator[int]:
×
384
        """Yields the build IDs where the crash occurred."""
385
        for build_id in self.signature["facets"]["build_id"]:
×
386
            yield build_id["term"]
×
387

388
    @property
×
389
    def top_build_id(self) -> int:
×
390
        """The build ID where most crashes occurred."""
391
        return self.signature["facets"]["build_id"][0]["term"]
×
392

393
    @cached_property
×
394
    def num_near_null_crashes(self) -> int:
×
395
        """The number of crashes that occurred near the null address."""
396
        return sum(
×
397
            address["count"]
398
            for address in self.signature["facets"]["address"]
399
            if is_near_null_address(address["term"])
400
        )
401

402
    @property
×
403
    def is_near_null_crash(self) -> bool:
×
404
        """Whether all crashes occurred near the null address."""
405
        return self.num_near_null_crashes == self.num_crashes
×
406

407
    @property
×
408
    def is_potential_near_null_crash(self) -> bool:
×
409
        """Whether the signature is a potential near null crash.
410

411
        The value will be True if some but not all crashes occurred near the
412
        null address.
413
        """
414
        return not self.is_near_null_crash and self.num_near_null_crashes > 0
×
415

416
    @property
×
417
    def is_near_null_related_crash(self) -> bool:
×
418
        """Whether the signature is related to near null crashes.
419

420
        The value will be True if any of the crashes occurred near the null
421
        address.
422
        """
423
        return self.is_near_null_crash or self.is_potential_near_null_crash
×
424

425
    @cached_property
×
426
    def num_near_allocator_crashes(self) -> int:
×
427
        """The number of crashes that occurred near an allocator address."""
428
        return sum(
×
429
            address["count"]
430
            for address in self.signature["facets"]["address"]
431
            if is_near_allocator_address(address["term"])
432
        )
433

434
    @property
×
435
    def is_near_allocator_crash(self) -> bool:
×
436
        """Whether all crashes occurred near an allocator address."""
437
        return self.num_near_allocator_crashes == self.num_crashes
×
438

439
    @property
×
440
    def is_potential_near_allocator_crash(self) -> bool:
×
441
        """Whether the signature is a potential near allocator crash.
442

443
        The value will be True if some but not all crashes occurred near an
444
        allocator address.
445
        """
446
        return not self.is_near_allocator_crash and self.num_near_allocator_crashes > 0
×
447

448
    @property
×
449
    def is_near_allocator_related_crash(self) -> bool:
×
450
        """Whether the signature is related to near allocator crashes.
451

452
        The value will be True if any of the crashes occurred near an allocator
453
        address.
454
        """
455
        return self.is_near_allocator_crash or self.is_potential_near_allocator_crash
×
456

457

458
class SignatureAnalyzer(SocorroDataAnalyzer, ClouseauDataAnalyzer):
×
459
    """Analyze the data related to a signature.
460

461
    This includes data from Socorro and Clouseau.
462
    """
463

464
    def __init__(
×
465
        self,
466
        socorro_signature: dict,
467
        num_total_crashes: int,
468
        clouseau_reports: list[dict],
469
    ):
470
        SocorroDataAnalyzer.__init__(self, socorro_signature, num_total_crashes)
×
471
        ClouseauDataAnalyzer.__init__(self, clouseau_reports)
×
472

473
    def _fetch_crash_reports(
×
474
        self,
475
        proto_signature: str,
476
        build_id: int | Iterable[int],
477
        limit: int = 1,
478
    ) -> Iterator[dict]:
479
        params = {
×
480
            "proto_signature": "=" + proto_signature,
481
            "build_id": build_id,
482
            "_columns": [
483
                "uuid",
484
            ],
485
            "_results_number": limit,
486
        }
487

488
        def handler(res: dict, data: dict):
×
489
            data.update(res)
×
490

491
        data: dict = {}
×
492
        socorro.SuperSearch(params=params, handler=handler, handlerdata=data).wait()
×
493

494
        yield from data["hits"]
×
495

496
    def fetch_representative_processed_crash(self) -> dict:
×
497
        """Fetch a processed crash to represent the signature.
498

499
        This could fetch multiple processed crashes and return the one that is
500
        most likely to be useful.
501
        """
502
        limit_to_top_proto_signature = (
×
503
            self.num_top_proto_signature_crashes / self.num_crashes > 0.6
504
        )
505

506
        reports = itertools.chain(
×
507
            # Reports with a higher score from clouseau are more likely to be
508
            # useful.
509
            sorted(
510
                self._clouseau_reports,
511
                key=lambda report: report["max_score"],
512
                reverse=True,
513
            ),
514
            # Next we try find reports from the top crashing build because they
515
            # are likely to be representative.
516
            self._fetch_crash_reports(self.top_proto_signature, self.top_build_id),
517
            self._fetch_crash_reports(self.top_proto_signature, self._build_ids()),
518
        )
519
        for report in reports:
×
520
            uuid = report["uuid"]
×
521
            processed_crash = socorro.ProcessedCrash.get_processed(uuid)[uuid]
×
522
            if (
×
523
                not limit_to_top_proto_signature
524
                or processed_crash["proto_signature"] == self.top_proto_signature
525
            ):
526
                # TODO(investigate): maybe we should check if the stack is
527
                # corrupted (ask gsvelto or willkg about how to detect that)
528
                return processed_crash
×
529

530
        raise NoCrashReportFoundError(
×
531
            f"No crash report found with the most frequent proto signature for {self.signature_term}."
532
        )
533

534

535
class SignaturesDataFetcher:
×
536
    """Fetch the data related to the given signatures."""
537

538
    MEMORY_ACCESS_ERROR_REASONS = (
×
539
        # On Windows:
540
        "EXCEPTION_ACCESS_VIOLATION_READ",
541
        "EXCEPTION_ACCESS_VIOLATION_WRITE",
542
        "EXCEPTION_ACCESS_VIOLATION_EXEC"
543
        # On Linux:
544
        "SIGSEGV / SEGV_MAPERR",
545
        "SIGSEGV / SEGV_ACCERR",
546
    )
547

548
    EXCLUDED_MOZ_REASON_STRINGS = (
×
549
        "MOZ_CRASH(OOM)",
550
        "MOZ_CRASH(Out of memory)",
551
        "out of memory",
552
        "Shutdown hanging",
553
        # TODO(investigate): do we need to exclude signatures that their reason
554
        # contains `[unhandlable oom]`?
555
        # Example: arena_t::InitChunk | arena_t::AllocRun | arena_t::MallocLarge | arena_t::Malloc | BaseAllocator::malloc | Allocator::malloc | PageMalloc
556
        # "[unhandlable oom]",
557
    )
558

559
    # If any of the crash reason starts with any of the following, then it is
560
    # Network or I/O error.
561
    EXCLUDED_IO_ERROR_REASON_PREFIXES = (
×
562
        "EXCEPTION_IN_PAGE_ERROR_READ",
563
        "EXCEPTION_IN_PAGE_ERROR_WRITE",
564
        "EXCEPTION_IN_PAGE_ERROR_EXEC",
565
    )
566

567
    # TODO(investigate): do we need to exclude all these signatures prefixes?
568
    EXCLUDED_SIGNATURE_PREFIXES = (
×
569
        "OOM | ",
570
        "bad hardware | ",
571
        "shutdownhang | ",
572
    )
573

574
    def __init__(
×
575
        self,
576
        signatures: Iterable[str],
577
        product: str = "Firefox",
578
        channel: str = "nightly",
579
    ):
580
        self._signatures = set(signatures)
×
581
        self._product = product
×
582
        self._channel = channel
×
583

584
    @classmethod
×
585
    def find_new_actionable_crashes(
×
586
        cls,
587
        product: str,
588
        channel: str,
589
        days_to_check: int = 7,
590
        days_without_crashes: int = 7,
591
    ) -> "SignaturesDataFetcher":
592
        """Find new actionable crashes.
593

594
        Args:
595
            product: The product to check.
596
            channel: The release channel to check.
597
            days_to_check: The number of days to check for crashes.
598
            days_without_crashes: The number of days without crashes before the
599
                `days_to_check` to consider the signature new.
600

601
        Returns:
602
            A list of actionable signatures.
603
        """
604
        duration = days_to_check + days_without_crashes
×
605
        end_date = lmdutils.get_date_ymd("today")
×
606
        start_date = end_date - timedelta(duration)
×
607
        earliest_allowed_date = lmdutils.get_date_str(
×
608
            end_date - timedelta(days_to_check)
609
        )
610
        date_range = socorro.SuperSearch.get_search_date(start_date, end_date)
×
611

612
        params = {
×
613
            "product": product,
614
            "release_channel": channel,
615
            "date": date_range,
616
            # TODO(investigate): should we do a local filter instead of the
617
            # following (should we exclude the signature if one of the crashes
618
            # is a shutdown hang?):
619
            # If the `ipc_shutdown_state` or `shutdown_progress` field are
620
            # non-empty then it's a shutdown hang.
621
            "ipc_shutdown_state": "__null__",
622
            "shutdown_progress": "__null__",
623
            # TODO(investigate): should we use the following instead of the
624
            # local filter.
625
            # "oom_allocation_size": "!__null__",
626
            "_aggs.signature": [
627
                "moz_crash_reason",
628
                "reason",
629
                "_histogram.date",
630
                "_cardinality.install_time",
631
                "_cardinality.oom_allocation_size",
632
            ],
633
            "_results_number": 0,
634
            "_facets_size": 10000,
635
        }
636

637
        def handler(search_resp: dict, data: list):
×
638
            logger.debug(
×
639
                "Total of %d signatures received from Socorro",
640
                len(search_resp["facets"]["signature"]),
641
            )
642

643
            for crash in search_resp["facets"]["signature"]:
×
644
                signature = crash["term"]
×
645
                if any(
×
646
                    signature.startswith(excluded_prefix)
647
                    for excluded_prefix in cls.EXCLUDED_SIGNATURE_PREFIXES
648
                ):
649
                    # Ignore signatures that start with any of the excluded prefixes.
650
                    continue
×
651

652
                facets = crash["facets"]
×
653
                installations = facets["cardinality_install_time"]["value"]
×
654
                if installations <= 1:
×
655
                    # Ignore crashes that only happen on one installation.
656
                    continue
×
657

658
                first_date = facets["histogram_date"][0]["term"]
×
659
                if first_date < earliest_allowed_date:
×
660
                    # The crash is not new, skip it.
661
                    continue
×
662

663
                if any(
×
664
                    reason["term"].startswith(io_error_prefix)
665
                    for reason in facets["reason"]
666
                    for io_error_prefix in cls.EXCLUDED_IO_ERROR_REASON_PREFIXES
667
                ):
668
                    # Ignore Network or I/O error crashes.
669
                    continue
×
670

671
                if crash["count"] < 20:
×
672
                    # For signatures with low volume, having multiple types of
673
                    # memory errors indicates potential bad hardware crashes.
674
                    num_memory_error_types = sum(
×
675
                        reason["term"] in cls.MEMORY_ACCESS_ERROR_REASONS
676
                        for reason in facets["reason"]
677
                    )
678
                    if num_memory_error_types > 1:
×
679
                        # Potential bad hardware crash, skip it.
680
                        continue
×
681

682
                # TODO: Add a filter using the `possible_bit_flips_max_confidence`
683
                # field to exclude bad hardware crashes. The filed is not available yet.
684
                # See: https://bugzilla.mozilla.org/show_bug.cgi?id=1816669#c3
685

686
                # TODO(investigate): is this needed since we are already
687
                # filtering signatures that start with "OOM | "
688
                if facets["cardinality_oom_allocation_size"]["value"]:
×
689
                    # If one of the crashes is an OOM crash, skip it.
690
                    continue
×
691

692
                # TODO(investigate): do we need to check for the `moz_crash_reason`
693
                moz_crash_reasons = facets["moz_crash_reason"]
×
694
                if moz_crash_reasons and any(
×
695
                    excluded_reason in reason["term"]
696
                    for reason in moz_crash_reasons
697
                    for excluded_reason in cls.EXCLUDED_MOZ_REASON_STRINGS
698
                ):
699
                    continue
×
700

701
                data.append(signature)
×
702

703
        signatures: list = []
×
704
        socorro.SuperSearch(
×
705
            params=params,
706
            handler=handler,
707
            handlerdata=signatures,
708
        ).wait()
709

710
        logger.debug(
×
711
            "Total of %d signatures left after applying the filtering criteria",
712
            len(signatures),
713
        )
714

715
        return cls(signatures, product, channel)
×
716

717
    def fetch_clouseau_crash_reports(self) -> dict[str, list]:
×
718
        """Fetch the crash reports data from Crash Clouseau."""
719
        signature_reports = clouseau.Reports.get_by_signatures(
×
720
            self._signatures,
721
            product=self._product,
722
            channel=self._channel,
723
        )
724

725
        logger.debug(
×
726
            "Total of %d signatures received from Clouseau", len(signature_reports)
727
        )
728

729
        return signature_reports
×
730

731
    def fetch_socorro_info(self) -> tuple[list[dict], int]:
×
732
        """Fetch the signature data from Socorro."""
733
        # TODO(investigate): should we increase the duration to 6 months?
734
        duration = timedelta(weeks=1)
×
735
        end_date = lmdutils.get_date_ymd("today")
×
736
        start_date = end_date - duration
×
737
        date_range = socorro.SuperSearch.get_search_date(start_date, end_date)
×
738

739
        params = {
×
740
            "product": self._product,
741
            # TODO(investigate): should we included all release channels?
742
            "release_channel": self._channel,
743
            # TODO(investigate): should we limit based on the build date as well?
744
            "date": date_range,
745
            # TODO: split signatures into chunks to avoid very long query URLs
746
            "signature": ["=" + signature for signature in self._signatures],
747
            "_aggs.signature": [
748
                "address",
749
                "build_id",
750
                "cpu_arch",
751
                "proto_signature",
752
                "_cardinality.user_comments",
753
                "cpu_arch",
754
                "platform_pretty_version",
755
                # The following are needed for SignatureStats:
756
                "platform",
757
                "is_garbage_collecting",
758
                "_cardinality.install_time",
759
                "startup_crash",
760
                "_histogram.uptime",
761
                "process_type",
762
            ],
763
            "_results_number": 0,
764
            "_facets_size": 10000,
765
        }
766

767
        def handler(search_results: dict, data: dict):
×
768
            data["num_total_crashes"] = search_results["total"]
×
769
            data["signatures"] = search_results["facets"]["signature"]
×
770

771
        data: dict = {}
×
772
        socorro.SuperSearchUnredacted(
×
773
            params=params,
774
            handler=handler,
775
            handlerdata=data,
776
        ).wait()
777

778
        logger.debug(
×
779
            "Fetch info from Socorro for %d signatures", len(data["signatures"])
780
        )
781

782
        return data["signatures"], data["num_total_crashes"]
×
783

784
    def fetch_bugs(self, include_fields: list[str] = None) -> dict[str, list[dict]]:
×
785
        """Fetch bugs that are filed against the given signatures."""
786

787
        params_base: dict = {
×
788
            "include_fields": [
789
                "cf_crash_signature",
790
            ],
791
        }
792

793
        if include_fields:
×
794
            params_base["include_fields"].extend(include_fields)
×
795

796
        params_list = []
×
797
        for signatures_chunk in Connection.chunks(list(self._signatures), 30):
×
798
            params = params_base.copy()
×
799
            n = int(utils.get_last_field_num(params))
×
800
            params[f"f{n}"] = "OP"
×
801
            params[f"j{n}"] = "OR"
×
802
            for signature in signatures_chunk:
×
803
                n += 1
×
804
                params[f"f{n}"] = "cf_crash_signature"
×
805
                params[f"o{n}"] = "regexp"
×
806
                params[f"v{n}"] = rf"\[(@ |@){re.escape(signature)}( \]|\])"
×
807
            params[f"f{n+1}"] = "CP"
×
808
            params_list.append(params)
×
809

810
        signatures_bugs: dict = defaultdict(list)
×
811

812
        def handler(res, data):
×
813
            for bug in res["bugs"]:
×
814
                for signature in utils.get_signatures(bug["cf_crash_signature"]):
×
815
                    if signature in self._signatures:
×
816
                        data[signature].append(bug)
×
817

818
        Bugzilla(
×
819
            queries=[
820
                connection.Query(Bugzilla.API_URL, params, handler, signatures_bugs)
821
                for params in params_list
822
            ],
823
        ).wait()
824

825
        # TODO: remove the call to DevBugzilla after moving to production
826
        DevBugzilla(
×
827
            queries=[
828
                connection.Query(DevBugzilla.API_URL, params, handler, signatures_bugs)
829
                for params in params_list
830
            ],
831
        ).wait()
832

833
        logger.debug(
×
834
            "Total of %d signatures already have bugs filed", len(signatures_bugs)
835
        )
836

837
        return signatures_bugs
×
838

839
    def analyze(self) -> list[SignatureAnalyzer]:
×
840
        """Analyze the data related to the signatures."""
841
        bugs = self.fetch_bugs()
×
842
        # TODO(investigate): For now, we are ignoring signatures that have bugs
843
        # filed even if they are closed long time ago. We should investigate
844
        # whether we should include the ones with closed bugs. For example, if
845
        # the bug was closed as Fixed years ago.
846
        self._signatures.difference_update(bugs.keys())
×
847

848
        clouseau_reports = self.fetch_clouseau_crash_reports()
×
849
        # TODO(investigate): For now, we are ignoring signatures that are not
850
        # analyzed by clouseau. We should investigate why they are not analyzed
851
        # and whether we should include them.
852
        self._signatures.intersection_update(clouseau_reports.keys())
×
853

854
        signatures, num_total_crashes = self.fetch_socorro_info()
×
855
        logger.debug("Total of %d signatures will be analyzed", len(signatures))
×
856

857
        return [
×
858
            SignatureAnalyzer(
859
                signature,
860
                num_total_crashes,
861
                clouseau_reports[signature["term"]],
862
            )
863
            for signature in signatures
864
        ]
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2025 Coveralls, Inc