• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

mozilla / relman-auto-nag / #4935

12 Feb 2024 01:59PM CUT coverage: 21.709%. Remained the same
#4935

push

coveralls-python

web-flow
Bump numpy from 1.26.3 to 1.26.4

Bumps [numpy](https://github.com/numpy/numpy) from 1.26.3 to 1.26.4.
- [Release notes](https://github.com/numpy/numpy/releases)
- [Changelog](https://github.com/numpy/numpy/blob/main/doc/RELEASE_WALKTHROUGH.rst)
- [Commits](https://github.com/numpy/numpy/compare/v1.26.3...v1.26.4)

---
updated-dependencies:
- dependency-name: numpy
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>

716 of 3614 branches covered (0.0%)

1928 of 8881 relevant lines covered (21.71%)

0.22 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0
/bugbot/crash/analyzer.py
1
# This Source Code Form is subject to the terms of the Mozilla Public
2
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
3
# You can obtain one at http://mozilla.org/MPL/2.0/.
4

5
import itertools
×
6
import re
×
7
from collections import defaultdict
×
8
from datetime import date, datetime, timedelta
×
9
from functools import cached_property
×
10
from typing import Iterable, Iterator
×
11

12
from dateutil import parser
×
13
from libmozdata import bugzilla, clouseau, connection, socorro
×
14
from libmozdata import utils as lmdutils
×
15
from libmozdata.bugzilla import Bugzilla
×
16
from libmozdata.connection import Connection
×
17

18
from bugbot import logger, utils
×
19
from bugbot.bug.analyzer import BugAnalyzer, BugsStore
×
20
from bugbot.components import ComponentName
×
21
from bugbot.crash import socorro_util
×
22

23
# The max offset from a memory address to be considered "near".
24
OFFSET_64_BIT = 0x1000
×
25
OFFSET_32_BIT = 0x100
×
26
# Allocator poison value addresses.
27
ALLOCATOR_ADDRESSES_64_BIT = (
×
28
    (0xE5E5E5E5E5E5E5E5, OFFSET_64_BIT),
29
    # On 64-bit windows, sometimes it could be doing something with a 32-bit
30
    # value gotten from freed memory, so it'll be 0X00000000E5E5E5E5 +/-, and
31
    # because of the address limitation, quite often it will be
32
    # 0X0000E5E5E5E5E5E5 +/-.
33
    (0x00000000E5E5E5E5, OFFSET_32_BIT),
34
    (0x0000E5E5E5E5E5E5, OFFSET_64_BIT),
35
    (0x4B4B4B4B4B4B4B4B, OFFSET_64_BIT),
36
)
37
ALLOCATOR_ADDRESSES_32_BIT = (
×
38
    (0xE5E5E5E5, OFFSET_32_BIT),
39
    (0x4B4B4B4B, OFFSET_32_BIT),
40
)
41
# Ranges where addresses are considered near allocator poison values.
42
ALLOCATOR_RANGES_64_BIT = tuple(
×
43
    (addr - offset, addr + offset) for addr, offset in ALLOCATOR_ADDRESSES_64_BIT
44
)
45
ALLOCATOR_RANGES_32_BIT = tuple(
×
46
    (addr - offset, addr + offset) for addr, offset in ALLOCATOR_ADDRESSES_32_BIT
47
)
48

49

50
def is_near_null_address(str_address) -> bool:
×
51
    """Check if the address is near null.
52

53
    Args:
54
        str_address: The memory address to check.
55

56
    Returns:
57
        True if the address is near null, False otherwise.
58
    """
59
    address = int(str_address, 0)
×
60
    is_64_bit = len(str_address) >= 18
×
61

62
    if is_64_bit:
×
63
        return -OFFSET_64_BIT <= address <= OFFSET_64_BIT
×
64

65
    return -OFFSET_32_BIT <= address <= OFFSET_32_BIT
×
66

67

68
def is_near_allocator_address(str_address) -> bool:
×
69
    """Check if the address is near an allocator poison value.
70

71
    Args:
72
        str_address: The memory address to check.
73

74
    Returns:
75
        True if the address is near an allocator poison value, False otherwise.
76
    """
77
    address = int(str_address, 0)
×
78
    is_64_bit = len(str_address) >= 18
×
79

80
    return any(
×
81
        low <= address <= high
82
        for low, high in (
83
            ALLOCATOR_RANGES_64_BIT if is_64_bit else ALLOCATOR_RANGES_32_BIT
84
        )
85
    )
86

87

88
# TODO: Move this to libmozdata
89
def generate_signature_page_url(params: dict, tab: str) -> str:
×
90
    """Generate a URL to the signature page on Socorro
91

92
    Args:
93
        params: the parameters for the search query.
94
        tab: the page tab that should be selected.
95

96
    Returns:
97
        The URL of the signature page on Socorro
98
    """
99
    web_url = socorro.Socorro.CRASH_STATS_URL
×
100
    query = lmdutils.get_params_for_url(params)
×
101
    return f"{web_url}/signature/{query}#{tab}"
×
102

103

104
class NoCrashReportFoundError(Exception):
×
105
    """There are no crash reports that meet the required criteria."""
106

107

108
class ClouseauDataAnalyzer:
×
109
    """Analyze the data returned by Crash Clouseau about a specific crash
110
    signature.
111
    """
112

113
    MINIMUM_CLOUSEAU_SCORE_THRESHOLD: int = 8
×
114
    DEFAULT_CRASH_COMPONENT = ComponentName("Core", "General")
×
115

116
    def __init__(
×
117
        self, reports: Iterable[dict], bugs_store: BugsStore, first_crash_date: datetime
118
    ):
119
        self._clouseau_reports = reports
×
120
        self._first_crash_date = first_crash_date
×
121
        self.bugs_store = bugs_store
×
122

123
    @cached_property
×
124
    def max_clouseau_score(self):
×
125
        """The maximum Clouseau score in the crash reports."""
126
        if not self._clouseau_reports:
×
127
            return 0
×
128
        return max(report["max_score"] for report in self._clouseau_reports)
×
129

130
    @cached_property
×
131
    def regressed_by_potential_bug_ids(self) -> set[int]:
×
132
        """The IDs for the bugs that their patches could have caused the crash."""
133
        return {
×
134
            changeset["bug_id"] for changeset in self.regressed_by_potential_patches
135
        }
136

137
    @cached_property
×
138
    def regressed_by_patch(self) -> str | None:
×
139
        """The hash of the patch that could have caused the crash."""
140
        potential_patches = {
×
141
            changeset["changeset"] for changeset in self.regressed_by_potential_patches
142
        }
143
        if len(potential_patches) == 1:
×
144
            return next(iter(potential_patches))
×
145
        return None
×
146

147
    @cached_property
×
148
    def regressed_by(self) -> int | None:
×
149
        """The ID of the bug that one of its patches could have caused
150
        the crash.
151

152
        If there are multiple bugs, the value will be `None`.
153
        """
154
        bug_ids = self.regressed_by_potential_bug_ids
×
155
        if len(bug_ids) == 1:
×
156
            return next(iter(bug_ids))
×
157
        return None
×
158

159
    @cached_property
×
160
    def regressed_by_potential_bugs(self) -> list[BugAnalyzer]:
×
161
        """The bugs whose patches could have caused the crash."""
162
        self.bugs_store.fetch_bugs(
×
163
            self.regressed_by_potential_bug_ids,
164
            [
165
                "id",
166
                "groups",
167
                "assigned_to",
168
                "product",
169
                "component",
170
                "_custom",
171
            ],
172
        )
173
        return [
×
174
            self.bugs_store.get_bug_by_id(bug_id)
175
            for bug_id in self.regressed_by_potential_bug_ids
176
        ]
177

178
    @cached_property
×
179
    def regressed_by_author(self) -> dict | None:
×
180
        """The author of the patch that could have caused the crash.
181

182
        If there are multiple regressors, the value will be `None`.
183

184
        The regressor bug assignee is considered as the author, even if the
185
        assignee is not the patch author.
186
        """
187

188
        if not self.regressed_by:
×
189
            return None
×
190

191
        bug = self.regressed_by_potential_bugs[0]
×
192
        assert bug.id == self.regressed_by
×
193
        return bug.get_field("assigned_to_detail")
×
194

195
    @cached_property
×
196
    def crash_component(self) -> ComponentName:
×
197
        """The component that the crash belongs to.
198

199
        If there are multiple components, the value will be the default one.
200
        """
201
        potential_components = {
×
202
            bug.component for bug in self.regressed_by_potential_bugs
203
        }
204
        if len(potential_components) == 1:
×
205
            return next(iter(potential_components))
×
206
        return self.DEFAULT_CRASH_COMPONENT
×
207

208
    @property
×
209
    def regressed_by_potential_patches(self) -> Iterator[dict]:
×
210
        """The patches that could have caused the crash.
211

212
        Example of a patch data:
213
            {
214
                "bug_id": 1668136,
215
                "changeset": "aa66fda02aac",
216
                "channel": "nightly",
217
                "is_backedout": False,
218
                "is_merge": False,
219
                "max_score": 0,
220
                "push_date": "Tue, 31 Oct 2023 09:30:58 GMT",
221
            }
222
        """
223
        minimum_accepted_score = max(
×
224
            self.MINIMUM_CLOUSEAU_SCORE_THRESHOLD, self.max_clouseau_score
225
        )
226
        return (
×
227
            changeset
228
            for report in self._clouseau_reports
229
            if report["max_score"] >= minimum_accepted_score
230
            for changeset in report["changesets"]
231
            if changeset["max_score"] >= minimum_accepted_score
232
            and not changeset["is_merge"]
233
            and not changeset["is_backedout"]
234
            # NOTE(marco): This aims to reduce noise but could exclude valid
235
            # regressors, such as when a single signature refers to multiple
236
            # crash causes.
237
            and self._first_crash_date > parser.parse(changeset["push_date"])
238
        )
239

240

241
class SocorroDataAnalyzer(socorro_util.SignatureStats):
×
242
    """Analyze the data returned by Socorro."""
243

244
    _bugzilla_os_legal_values = None
×
245
    _bugzilla_cpu_legal_values_map = None
×
246
    _platforms = [
×
247
        {"short_name": "win", "name": "Windows"},
248
        {"short_name": "mac", "name": "Mac OS X"},
249
        {"short_name": "lin", "name": "Linux"},
250
        {"short_name": "and", "name": "Android"},
251
        {"short_name": "unknown", "name": "Unknown"},
252
    ]
253

254
    def __init__(
×
255
        self,
256
        signature: dict,
257
        num_total_crashes: int,
258
    ):
259
        super().__init__(signature, num_total_crashes, platforms=self._platforms)
×
260

261
    @classmethod
×
262
    def to_bugzilla_op_sys(cls, op_sys: str) -> str:
×
263
        """Return the corresponding OS name in Bugzilla for the provided OS name
264
        from Socorro.
265

266
        If the OS name is not recognized, return "Other".
267
        """
268
        if cls._bugzilla_os_legal_values is None:
×
269
            cls._bugzilla_os_legal_values = set(
×
270
                bugzilla.BugFields.fetch_field_values("op_sys")
271
            )
272

273
        if op_sys in cls._bugzilla_os_legal_values:
×
274
            return op_sys
×
275

276
        if op_sys.startswith("OS X ") or op_sys.startswith("macOS "):
×
277
            op_sys = "macOS"
×
278
        elif op_sys.startswith("Windows"):
×
279
            op_sys = "Windows"
×
280
        elif "Linux" in op_sys or op_sys.startswith("Ubuntu"):
×
281
            op_sys = "Linux"
×
282
        else:
283
            op_sys = "Other"
×
284

285
        return op_sys
×
286

287
    @cached_property
×
288
    def first_crash_date(self) -> datetime:
×
289
        """The date of the first crash within the query time range."""
290
        return parser.parse(self.signature["facets"]["histogram_date"][0]["term"])
×
291

292
    @property
×
293
    def first_crash_date_ymd(self) -> str:
×
294
        """The date of the first crash within the query time range.
295

296
        The date is in YYYY-MM-DD format.
297
        """
298
        return self.first_crash_date.strftime("%Y-%m-%d")
×
299

300
    @property
×
301
    def bugzilla_op_sys(self) -> str:
×
302
        """The name of the OS where the crash happens.
303

304
        The value is one of the legal values for Bugzilla's `op_sys` field.
305

306
        - If no OS name is found, the value will be "Unspecified".
307
        - If the OS name is not recognized, the value will be "Other".
308
        - If multiple OS names are found, the value will be "All". Unless the OS
309
          names can be resolved to a common name without a version. For example,
310
          "Windows 10" and "Windows 7" will become "Windows".
311
        """
312
        all_op_sys = {
×
313
            self.to_bugzilla_op_sys(op_sys["term"])
314
            for op_sys in self.signature["facets"]["platform_pretty_version"]
315
        }
316

317
        if len(all_op_sys) > 1:
×
318
            # Resolve to root OS name by removing the version number.
319
            all_op_sys = {op_sys.split(" ")[0] for op_sys in all_op_sys}
×
320

321
        if len(all_op_sys) == 2 and "Other" in all_op_sys:
×
322
            # TODO: explain this workaround.
323
            all_op_sys.remove("Other")
×
324

325
        if len(all_op_sys) == 1:
×
326
            return next(iter(all_op_sys))
×
327

328
        if len(all_op_sys) == 0:
×
329
            return "Unspecified"
×
330

331
        return "All"
×
332

333
    @classmethod
×
334
    def to_bugzilla_cpu(cls, cpu: str) -> str:
×
335
        """Return the corresponding CPU name in Bugzilla for the provided name
336
        from Socorro.
337

338
        If the CPU is not recognized, return "Other".
339
        """
340
        if cls._bugzilla_cpu_legal_values_map is None:
×
341
            cls._bugzilla_cpu_legal_values_map = {
×
342
                value.lower(): value
343
                for value in bugzilla.BugFields.fetch_field_values("rep_platform")
344
            }
345

346
        return cls._bugzilla_cpu_legal_values_map.get(cpu, "Other")
×
347

348
    @property
×
349
    def bugzilla_cpu_arch(self) -> str:
×
350
        """The CPU architecture of the devices where the crash happens.
351

352
        The value is one of the legal values for Bugzilla's `rep_platform` field.
353

354
        - If no CPU architecture is found, the value will be "Unspecified".
355
        - If the CPU architecture is not recognized, the value will be "Other".
356
        - If multiple CPU architectures are found, the value will "All".
357
        """
358
        all_cpu_arch = {
×
359
            self.to_bugzilla_cpu(cpu["term"])
360
            for cpu in self.signature["facets"]["cpu_arch"]
361
        }
362

363
        if len(all_cpu_arch) == 2 and "Other" in all_cpu_arch:
×
364
            all_cpu_arch.remove("Other")
×
365

366
        if len(all_cpu_arch) == 1:
×
367
            return next(iter(all_cpu_arch))
×
368

369
        if len(all_cpu_arch) == 0:
×
370
            return "Unspecified"
×
371

372
        return "All"
×
373

374
    @property
×
375
    def user_comments_page_url(self) -> str:
×
376
        """The URL to the Signature page on Socorro where the Comments tab is
377
        selected.
378
        """
379
        start_date = date.today() - timedelta(weeks=26)
×
380
        params = {
×
381
            "signature": self.signature_term,
382
            "date": socorro.SuperSearch.get_search_date(start_date),
383
        }
384
        return generate_signature_page_url(params, "comments")
×
385

386
    @property
×
387
    def num_user_comments(self) -> int:
×
388
        """The number of crash reports with user comments."""
389
        # TODO: count useful/interesting user comments (e.g., exclude one word comments)
390
        return self.signature["facets"]["cardinality_user_comments"]["value"]
×
391

392
    @property
×
393
    def has_user_comments(self) -> bool:
×
394
        """Whether the crash signature has any reports with a user comment."""
395
        return self.num_user_comments > 0
×
396

397
    @property
×
398
    def top_proto_signature(self) -> str:
×
399
        """The proto signature that occurs the most."""
400
        return self.signature["facets"]["proto_signature"][0]["term"]
×
401

402
    @property
×
403
    def num_top_proto_signature_crashes(self) -> int:
×
404
        """The number of crashes for the most occurring proto signature."""
405
        return self.signature["facets"]["proto_signature"][0]["count"]
×
406

407
    def _build_ids(self) -> Iterator[int]:
×
408
        """Yields the build IDs where the crash occurred."""
409
        for build_id in self.signature["facets"]["build_id"]:
×
410
            yield build_id["term"]
×
411

412
    @property
×
413
    def top_build_id(self) -> int:
×
414
        """The build ID where most crashes occurred."""
415
        return self.signature["facets"]["build_id"][0]["term"]
×
416

417
    @cached_property
×
418
    def num_near_null_crashes(self) -> int:
×
419
        """The number of crashes that occurred on addresses near null."""
420
        return sum(
×
421
            address["count"]
422
            for address in self.signature["facets"]["address"]
423
            if is_near_null_address(address["term"])
424
        )
425

426
    @property
×
427
    def is_near_null_crash(self) -> bool:
×
428
        """Whether all crashes occurred on addresses near null."""
429
        return self.num_near_null_crashes == self.num_crashes
×
430

431
    @property
×
432
    def is_potential_near_null_crash(self) -> bool:
×
433
        """Whether the signature is a potential near null crash.
434

435
        The value will be True if some but not all crashes occurred on addresses
436
        near null.
437
        """
438
        return not self.is_near_null_crash and self.num_near_null_crashes > 0
×
439

440
    @property
×
441
    def is_near_null_related_crash(self) -> bool:
×
442
        """Whether the signature is related to near null crashes.
443

444
        The value will be True if any of the crashes occurred on addresses near
445
        null.
446
        """
447
        return self.is_near_null_crash or self.is_potential_near_null_crash
×
448

449
    @cached_property
×
450
    def num_near_allocator_crashes(self) -> int:
×
451
        """The number of crashes that occurred on addresses near an allocator
452
        poison value.
453
        """
454
        return sum(
×
455
            address["count"]
456
            for address in self.signature["facets"]["address"]
457
            if is_near_allocator_address(address["term"])
458
        )
459

460
    @property
×
461
    def is_near_allocator_crash(self) -> bool:
×
462
        """Whether all crashes occurred on addresses near an allocator poison
463
        value.
464
        """
465
        return self.num_near_allocator_crashes == self.num_crashes
×
466

467
    @property
×
468
    def is_potential_near_allocator_crash(self) -> bool:
×
469
        """Whether the signature is a potential near allocator poison value
470
        crash.
471

472
        The value will be True if some but not all crashes occurred on addresses
473
        near an allocator poison value.
474
        """
475
        return not self.is_near_allocator_crash and self.num_near_allocator_crashes > 0
×
476

477
    @property
×
478
    def is_near_allocator_related_crash(self) -> bool:
×
479
        """Whether the signature is related to near allocator poison value
480
        crashes.
481

482
        The value will be True if any of the crashes occurred on addresses near
483
        an allocator poison value.
484
        """
485
        return self.is_near_allocator_crash or self.is_potential_near_allocator_crash
×
486

487
    @cached_property
×
488
    def num_phc_crashes(self) -> int:
×
489
        """The number of crashes that are related to a potential Probabilistic
490
        Heap Checker (PHC) bug.
491
        """
492
        return sum(
×
493
            crash["count"] for crash in self.signature["facets"]["phc_alloc_stack"]
494
        )
495

496
    @property
×
497
    def is_potential_phc_crash(self) -> bool:
×
498
        """Whether the crash is related to a potential Probabilistic Heap
499
        Checker (PHC) bug.
500
        """
501
        return self.num_phc_crashes > 0
×
502

503
    @property
×
504
    def is_phc_crash(self) -> bool:
×
505
        """Whether the crash is related to a potential Probabilistic Heap
506
        Checker (PHC) bug.
507
        """
508
        return self.num_phc_crashes == self.num_crashes
×
509

510

511
class SignatureAnalyzer(SocorroDataAnalyzer, ClouseauDataAnalyzer):
×
512
    """Analyze the data related to a signature.
513

514
    This includes data from Socorro and Clouseau.
515
    """
516

517
    def __init__(
×
518
        self,
519
        socorro_signature: dict,
520
        num_total_crashes: int,
521
        clouseau_reports: list[dict],
522
        bugs_store: BugsStore,
523
    ):
524
        SocorroDataAnalyzer.__init__(self, socorro_signature, num_total_crashes)
×
525
        ClouseauDataAnalyzer.__init__(
×
526
            self, clouseau_reports, bugs_store, self.first_crash_date
527
        )
528

529
    def _fetch_crash_reports(
×
530
        self,
531
        proto_signature: str,
532
        build_id: int | Iterable[int],
533
        limit: int = 1,
534
    ) -> Iterator[dict]:
535
        params = {
×
536
            "proto_signature": "=" + proto_signature,
537
            "build_id": build_id,
538
            "_columns": [
539
                "uuid",
540
            ],
541
            "_results_number": limit,
542
        }
543

544
        def handler(res: dict, data: dict):
×
545
            data.update(res)
×
546

547
        data: dict = {}
×
548
        socorro.SuperSearch(params=params, handler=handler, handlerdata=data).wait()
×
549

550
        yield from data["hits"]
×
551

552
    def _is_corrupted_crash_stack(self, processed_crash: dict) -> bool:
×
553
        """Whether the crash stack is corrupted.
554

555
        Args:
556
            processed_crash: The processed crash to check.
557

558
        Returns:
559
            True if the crash stack is corrupted, False otherwise.
560
        """
561

562
        return any(
×
563
            not frame["module"]
564
            for frame in processed_crash["json_dump"]["crashing_thread"]["frames"]
565
        )
566

567
    def fetch_representative_processed_crash(self) -> dict:
×
568
        """Fetch a processed crash to represent the signature.
569

570
        This could fetch multiple processed crashes and return the one that is
571
        most likely to be useful.
572
        """
573
        limit_to_top_proto_signature = (
×
574
            self.num_top_proto_signature_crashes / self.num_crashes > 0.6
575
        )
576

577
        candidate_reports = itertools.chain(
×
578
            # Reports with a higher score from clouseau are more likely to be
579
            # useful.
580
            sorted(
581
                self._clouseau_reports,
582
                key=lambda report: report["max_score"],
583
                reverse=True,
584
            ),
585
            # Next we try find reports from the top crashing build because they
586
            # are likely to be representative.
587
            self._fetch_crash_reports(self.top_proto_signature, self.top_build_id),
588
            self._fetch_crash_reports(self.top_proto_signature, self._build_ids()),
589
        )
590

591
        first_representative_report = None
×
592
        for i, report in enumerate(candidate_reports):
×
593
            uuid = report["uuid"]
×
594
            processed_crash = socorro.ProcessedCrash.get_processed(uuid)[uuid]
×
595
            if (
×
596
                limit_to_top_proto_signature
597
                and processed_crash["proto_signature"] != self.top_proto_signature
598
            ):
599
                continue
×
600

601
            if first_representative_report is None:
×
602
                first_representative_report = processed_crash
×
603

604
            if not self._is_corrupted_crash_stack(processed_crash):
×
605
                return processed_crash
×
606

607
            if i >= 20:
×
608
                # We have tried enough reports, give up.
609
                break
×
610

611
        if first_representative_report is not None:
×
612
            # Fall back to the first representative report that we found, even
613
            # if it's corrupted.
614
            return first_representative_report
×
615

616
        raise NoCrashReportFoundError(
×
617
            f"No crash report found with the most frequent proto signature for {self.signature_term}."
618
        )
619

620
    @cached_property
×
621
    def is_potential_security_crash(self) -> bool:
×
622
        """Whether the crash is related to a potential security bug."""
623
        return (
×
624
            self.is_near_allocator_related_crash
625
            or self.is_potential_phc_crash
626
            or any(bug.is_security for bug in self.regressed_by_potential_bugs)
627
        )
628

629
    def has_moz_crash_reason(self, reason: str) -> bool:
×
630
        """Whether the crash has a specific MOZ_CRASH reason.
631

632
        Args:
633
            reason: The MOZ_CRASH reason to check.
634

635
        Returns:
636
            True if the any of the MOZ_CRASH reasons has a partial match with
637
            the provided reason.
638
        """
639
        return any(
×
640
            reason in moz_crash_reason["term"]
641
            for moz_crash_reason in self.signature["facets"]["moz_crash_reason"]
642
        )
643

644
    @property
×
645
    def process_type_summary(self) -> str:
×
646
        """The summary of the process types for the crash signature."""
647
        process_types = self.signature["facets"]["process_type"]
×
648
        if len(process_types) == 0:
×
649
            return "Unknown"
×
650

651
        if len(process_types) == 1:
×
652
            process_type = process_types[0]["term"]
×
653
            # Small process types are usually acronyms (e.g., gpu for GPU), thus
654
            # we use upper case for them. Otherwise, we capitalize the first letter.
655
            if len(process_type) <= 3:
×
656
                return process_type.upper()
×
657
            return process_type.capitalize()
×
658

659
        return "Multiple distinct types"
×
660

661

662
class SignaturesDataFetcher:
×
663
    """Fetch the data related to the given signatures."""
664

665
    MEMORY_ACCESS_ERROR_REASONS = (
×
666
        # On Windows:
667
        "EXCEPTION_ACCESS_VIOLATION_READ",
668
        "EXCEPTION_ACCESS_VIOLATION_WRITE",
669
        "EXCEPTION_ACCESS_VIOLATION_EXEC"
670
        # On Linux:
671
        "SIGSEGV / SEGV_MAPERR",
672
        "SIGSEGV / SEGV_ACCERR",
673
    )
674

675
    EXCLUDED_MOZ_REASON_STRINGS = (
×
676
        "MOZ_CRASH(OOM)",
677
        "MOZ_CRASH(Out of memory)",
678
        "out of memory",
679
        "Shutdown hanging",
680
        # TODO(investigate): do we need to exclude signatures that their reason
681
        # contains `[unhandlable oom]`?
682
        # Example: arena_t::InitChunk | arena_t::AllocRun | arena_t::MallocLarge | arena_t::Malloc | BaseAllocator::malloc | Allocator::malloc | PageMalloc
683
        # "[unhandlable oom]",
684
    )
685

686
    # If any of the crash reason starts with any of the following, then it is
687
    # Network or I/O error.
688
    EXCLUDED_IO_ERROR_REASON_PREFIXES = (
×
689
        "EXCEPTION_IN_PAGE_ERROR_READ",
690
        "EXCEPTION_IN_PAGE_ERROR_WRITE",
691
        "EXCEPTION_IN_PAGE_ERROR_EXEC",
692
    )
693

694
    # TODO(investigate): do we need to exclude all these signatures prefixes?
695
    EXCLUDED_SIGNATURE_PREFIXES = (
×
696
        "OOM | ",
697
        "bad hardware | ",
698
        "shutdownhang | ",
699
    )
700

701
    SUMMARY_DURATION = timedelta(weeks=10)
×
702

703
    def __init__(
×
704
        self,
705
        signatures: Iterable[str],
706
        product: str = "Firefox",
707
        channel: str = "nightly",
708
    ):
709
        self._signatures = set(signatures)
×
710
        self._product = product
×
711
        self._channel = channel
×
712

713
    @classmethod
×
714
    def find_new_actionable_crashes(
×
715
        cls,
716
        product: str,
717
        channel: str,
718
        days_to_check: int = 7,
719
        days_without_crashes: int = 7,
720
    ) -> "SignaturesDataFetcher":
721
        """Find new actionable crashes.
722

723
        Args:
724
            product: The product to check.
725
            channel: The release channel to check.
726
            days_to_check: The number of days to check for crashes.
727
            days_without_crashes: The number of days without crashes before the
728
                `days_to_check` to consider the signature new.
729

730
        Returns:
731
            A list of actionable signatures.
732
        """
733
        duration = days_to_check + days_without_crashes
×
734
        end_date = lmdutils.get_date_ymd("today")
×
735
        start_date = end_date - timedelta(duration)
×
736
        earliest_allowed_date = lmdutils.get_date_str(
×
737
            end_date - timedelta(days_to_check)
738
        )
739
        date_range = socorro.SuperSearch.get_search_date(start_date, end_date)
×
740

741
        params = {
×
742
            "product": product,
743
            "release_channel": channel,
744
            "date": date_range,
745
            # TODO(investigate): should we do a local filter instead of the
746
            # following (should we exclude the signature if one of the crashes
747
            # is a shutdown hang?):
748
            # If the `ipc_shutdown_state` or `shutdown_progress` field are
749
            # non-empty then it's a shutdown hang.
750
            "ipc_shutdown_state": "__null__",
751
            "shutdown_progress": "__null__",
752
            # TODO(investigate): should we use the following instead of the
753
            # local filter.
754
            # "oom_allocation_size": "!__null__",
755
            "_aggs.signature": [
756
                "moz_crash_reason",
757
                "reason",
758
                "possible_bit_flips_max_confidence",
759
                "_histogram.date",
760
                "_cardinality.install_time",
761
                "_cardinality.oom_allocation_size",
762
            ],
763
            "_results_number": 0,
764
            "_facets_size": 10000,
765
        }
766

767
        def handler(search_resp: dict, data: list):
×
768
            logger.debug(
×
769
                "Total of %d signatures received from Socorro",
770
                len(search_resp["facets"]["signature"]),
771
            )
772

773
            for crash in search_resp["facets"]["signature"]:
×
774
                signature = crash["term"]
×
775
                if any(
×
776
                    signature.startswith(excluded_prefix)
777
                    for excluded_prefix in cls.EXCLUDED_SIGNATURE_PREFIXES
778
                ):
779
                    # Ignore signatures that start with any of the excluded prefixes.
780
                    continue
×
781

782
                facets = crash["facets"]
×
783
                installations = facets["cardinality_install_time"]["value"]
×
784
                if installations <= 1:
×
785
                    # Ignore crashes that only happen on one installation.
786
                    continue
×
787

788
                first_date = facets["histogram_date"][0]["term"]
×
789
                if first_date < earliest_allowed_date:
×
790
                    # The crash is not new, skip it.
791
                    continue
×
792

793
                if any(
×
794
                    reason["term"].startswith(io_error_prefix)
795
                    for reason in facets["reason"]
796
                    for io_error_prefix in cls.EXCLUDED_IO_ERROR_REASON_PREFIXES
797
                ):
798
                    # Ignore Network or I/O error crashes.
799
                    continue
×
800

801
                if crash["count"] < 20:
×
802
                    # For signatures with low volume, having multiple types of
803
                    # memory errors indicates potential bad hardware crashes.
804
                    num_memory_error_types = sum(
×
805
                        reason["term"] in cls.MEMORY_ACCESS_ERROR_REASONS
806
                        for reason in facets["reason"]
807
                    )
808
                    if num_memory_error_types > 1:
×
809
                        # Potential bad hardware crash, skip it.
810
                        continue
×
811

812
                bit_flips_count = sum(
×
813
                    row["count"] for row in facets["possible_bit_flips_max_confidence"]
814
                )
815
                bit_flips_percentage = bit_flips_count / crash["count"]
×
816
                if bit_flips_percentage >= 0.2:
×
817
                    # Potential bad hardware crash, skip it.
818
                    continue
×
819

820
                # TODO(investigate): is this needed since we are already
821
                # filtering signatures that start with "OOM | "
822
                if facets["cardinality_oom_allocation_size"]["value"]:
×
823
                    # If one of the crashes is an OOM crash, skip it.
824
                    continue
×
825

826
                # TODO(investigate): do we need to check for the `moz_crash_reason`
827
                moz_crash_reasons = facets["moz_crash_reason"]
×
828
                if moz_crash_reasons and any(
×
829
                    excluded_reason in reason["term"]
830
                    for reason in moz_crash_reasons
831
                    for excluded_reason in cls.EXCLUDED_MOZ_REASON_STRINGS
832
                ):
833
                    continue
×
834

835
                data.append(signature)
×
836

837
        signatures: list = []
×
838
        socorro.SuperSearch(
×
839
            params=params,
840
            handler=handler,
841
            handlerdata=signatures,
842
        ).wait()
843

844
        logger.debug(
×
845
            "Total of %d signatures left after applying the filtering criteria",
846
            len(signatures),
847
        )
848

849
        return cls(signatures, product, channel)
×
850

851
    def fetch_clouseau_crash_reports(self) -> dict[str, list]:
×
852
        """Fetch the crash reports data from Crash Clouseau."""
853
        if not self._signatures:
×
854
            return {}
×
855

856
        logger.debug(
×
857
            "Fetch from Clouseau: requesting reports for %d signatures",
858
            len(self._signatures),
859
        )
860

861
        signature_reports = clouseau.Reports.get_by_signatures(
×
862
            self._signatures,
863
            product=self._product,
864
            channel=self._channel,
865
        )
866

867
        logger.debug(
×
868
            "Fetch from Clouseau: received reports for %d signatures",
869
            len(signature_reports),
870
        )
871

872
        return signature_reports
×
873

874
    def fetch_socorro_info(self) -> tuple[list[dict], int]:
×
875
        """Fetch the signature data from Socorro."""
876
        if not self._signatures:
×
877
            return [], 0
×
878

879
        end_date = lmdutils.get_date_ymd("today")
×
880
        start_date = end_date - self.SUMMARY_DURATION
×
881
        date_range = socorro.SuperSearch.get_search_date(start_date, end_date)
×
882

883
        params = {
×
884
            "product": self._product,
885
            # TODO(investigate): should we included all release channels?
886
            "release_channel": self._channel,
887
            # TODO(investigate): should we limit based on the build date as well?
888
            "date": date_range,
889
            # TODO: split signatures into chunks to avoid very long query URLs
890
            "signature": ["=" + signature for signature in self._signatures],
891
            "_aggs.signature": [
892
                "address",
893
                "build_id",
894
                "cpu_arch",
895
                "proto_signature",
896
                "_cardinality.user_comments",
897
                "cpu_arch",
898
                "platform_pretty_version",
899
                "_histogram.date",
900
                "phc_alloc_stack",
901
                # The following are needed for SignatureStats:
902
                "platform",
903
                "is_garbage_collecting",
904
                "_cardinality.install_time",
905
                "startup_crash",
906
                "_histogram.uptime",
907
                "process_type",
908
                "moz_crash_reason",
909
            ],
910
            "_results_number": 0,
911
            "_facets_size": 10000,
912
        }
913

914
        def handler(search_results: dict, data: dict):
×
915
            data["num_total_crashes"] = search_results["total"]
×
916
            data["signatures"] = search_results["facets"]["signature"]
×
917

918
        logger.debug(
×
919
            "Fetch from Socorro: requesting info for %d signatures",
920
            len(self._signatures),
921
        )
922

923
        data: dict = {}
×
924
        socorro.SuperSearchUnredacted(
×
925
            params=params,
926
            handler=handler,
927
            handlerdata=data,
928
        ).wait()
929

930
        logger.debug(
×
931
            "Fetch from Socorro: received info for %d signatures",
932
            len(data["signatures"]),
933
        )
934

935
        return data["signatures"], data["num_total_crashes"]
×
936

937
    def fetch_bugs(
×
938
        self, include_fields: list[str] | None = None
939
    ) -> dict[str, list[dict]]:
940
        """Fetch bugs that are filed against the given signatures."""
941
        if not self._signatures:
×
942
            return {}
×
943

944
        params_base: dict = {
×
945
            "include_fields": [
946
                "cf_crash_signature",
947
            ],
948
        }
949

950
        if include_fields:
×
951
            params_base["include_fields"].extend(include_fields)
×
952

953
        params_list = []
×
954
        for signatures_chunk in Connection.chunks(list(self._signatures), 30):
×
955
            params = params_base.copy()
×
956
            n = int(utils.get_last_field_num(params))
×
957
            params[f"f{n}"] = "OP"
×
958
            params[f"j{n}"] = "OR"
×
959
            for signature in signatures_chunk:
×
960
                n += 1
×
961
                params[f"f{n}"] = "cf_crash_signature"
×
962
                params[f"o{n}"] = "regexp"
×
963
                params[f"v{n}"] = rf"\[(@ |@){re.escape(signature)}( \]|\])"
×
964
            params[f"f{n+1}"] = "CP"
×
965
            params_list.append(params)
×
966

967
        signatures_bugs: dict = defaultdict(list)
×
968

969
        def handler(res, data):
×
970
            for bug in res["bugs"]:
×
971
                for signature in utils.get_signatures(bug["cf_crash_signature"]):
×
972
                    if signature in self._signatures:
×
973
                        data[signature].append(bug)
×
974

975
        logger.debug(
×
976
            "Fetch from Bugzilla: requesting bugs for %d signatures",
977
            len(self._signatures),
978
        )
979
        timeout = utils.get_config("common", "bz_query_timeout")
×
980
        Bugzilla(
×
981
            timeout=timeout,
982
            queries=[
983
                connection.Query(Bugzilla.API_URL, params, handler, signatures_bugs)
984
                for params in params_list
985
            ],
986
        ).wait()
987

988
        logger.debug(
×
989
            "Fetch from Bugzilla: received bugs for %d signatures", len(signatures_bugs)
990
        )
991

992
        return signatures_bugs
×
993

994
    def analyze(self) -> list[SignatureAnalyzer]:
×
995
        """Analyze the data related to the signatures."""
996
        bugs = self.fetch_bugs()
×
997
        # TODO(investigate): For now, we are ignoring signatures that have bugs
998
        # filed even if they are closed long time ago. We should investigate
999
        # whether we should include the ones with closed bugs. For example, if
1000
        # the bug was closed as Fixed years ago.
1001
        self._signatures.difference_update(bugs.keys())
×
1002

1003
        clouseau_reports = self.fetch_clouseau_crash_reports()
×
1004
        # TODO(investigate): For now, we are ignoring signatures that are not
1005
        # analyzed by clouseau. We should investigate why they are not analyzed
1006
        # and whether we should include them.
1007
        self._signatures.intersection_update(clouseau_reports.keys())
×
1008

1009
        signatures, num_total_crashes = self.fetch_socorro_info()
×
1010
        bugs_store = BugsStore()
×
1011

1012
        return [
×
1013
            SignatureAnalyzer(
1014
                signature,
1015
                num_total_crashes,
1016
                clouseau_reports[signature["term"]],
1017
                bugs_store,
1018
            )
1019
            for signature in signatures
1020
        ]
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2025 Coveralls, Inc