1f27d7ef-045c-4b13-be21-3a6fa9a7e5a6

Committed 11 Apr 2025 10:13PM CUT coverage: 85.212% (+0.01%) from 85.201%

Build # 1f27d7ef-045c-4b13-be21-3a6fa9a7e5a6

Build Type

Pull #5500

circleci

Committed by

groovecoder

Commit Message

MPP-4012 - feat(glean): log API access as Glean server event

Introduce a new `api.accessed` Glean event to capture accesses to Relay API
endpoints. This includes the HTTP method and endpoint path, and logs events
for all `/api/` prefixed routes via a new middleware component.

- Added `record_api_accessed()` to `EventsServerEventLogger`
- Extended `RelayGleanLogger` with `log_api_accessed()` for easier integration
- Registered `GleanApiAccessMiddleware` to log access for all API routes
- Added corresponding unit test for API access logging
- Updated `relay-server-metrics.yaml` to define the `api.accessed` metric
- Updated notification email for several existing metrics to use relay-team@mozilla.com

Pull Request Pull Request #5500: WIP: MPP-4012 - feat(glean): log API access as Glean server event

Run Details

2461 of 3597 branches covered (68.42%)

Branch coverage included in aggregate %.

38 of 39 new or added lines in 5 files covered. (97.44%)

1 existing line in 1 file now uncovered.

17258 of 19544 relevant lines covered (88.3%)

9.83 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

95.96

/privaterelay/tests/utils.py

"""Helper functions for tests"""

import json
import random
from datetime import UTC, datetime
from logging import LogRecord
from typing import Any
from unittest._log import _LoggingWatcher
from uuid import uuid4

from django.conf import settings
from django.contrib.auth.models import User

import pytest
from allauth.socialaccount.models import SocialAccount
from model_bakery import baker


def make_free_test_user(email: str = "") -> User:
    """Make a user who has signed up for the free Relay plan."""
    if email:
        user = baker.make(User, email=email)
    else:
        user = baker.make(User)
    user.profile.server_storage = True
    user.profile.save()
    uid = str(uuid4())
    baker.make(
        SocialAccount,
        user=user,
        uid=uid,
        provider="fxa",
        extra_data={"avatar": "avatar.png", "uid": uid},
    )
    return user


def make_premium_test_user() -> User:
    """Make a user who has the premium Relay plan, but hasn't picked a subdomain."""
    premium_user = baker.make(User, email="premium@email.com")
    premium_user.profile.server_storage = True
    premium_user.profile.date_subscribed = datetime.now(tz=UTC)
    premium_user.profile.save()
    upgrade_test_user_to_premium(premium_user)
    return premium_user


def upgrade_test_user_to_premium(user: User) -> None:
    """Create an FxA SocialAccount with an unlimited email masks plan."""
    if SocialAccount.objects.filter(user=user).exists():
        raise Exception("upgrade_test_user_to_premium does not (yet) handle this.")
    uid = str(uuid4())
    baker.make(
        SocialAccount,
        user=user,
        uid=uid,
        provider="fxa",
        extra_data={
            "avatar": "avatar.png",
            "subscriptions": [premium_subscription()],
            "uid": uid,
        },
    )


def make_storageless_test_user() -> User:
    storageless_user = baker.make(User)
    storageless_user_profile = storageless_user.profile
    storageless_user_profile.server_storage = False
    storageless_user_profile.subdomain = "mydomain"
    storageless_user_profile.date_subscribed = datetime.now(tz=UTC)
    storageless_user_profile.save()
    upgrade_test_user_to_premium(storageless_user)
    return storageless_user


def premium_subscription() -> str:
    """Return a Mozilla account subscription that provides unlimited emails"""
    assert settings.SUBSCRIPTIONS_WITH_UNLIMITED
    premium_only_plans = list(
        set(settings.SUBSCRIPTIONS_WITH_UNLIMITED)
        - set(settings.SUBSCRIPTIONS_WITH_PHONE)
        - set(settings.SUBSCRIPTIONS_WITH_VPN)
    )
    assert premium_only_plans
    return random.choice(premium_only_plans)


def phone_subscription() -> str:
    """Return a Mozilla account subscription that provides a phone mask"""
    assert settings.SUBSCRIPTIONS_WITH_PHONE
    phones_only_plans = list(
        set(settings.SUBSCRIPTIONS_WITH_PHONE)
        - set(settings.SUBSCRIPTIONS_WITH_VPN)
        - set(settings.SUBSCRIPTIONS_WITH_UNLIMITED)
    )
    assert phones_only_plans
    return random.choice(phones_only_plans)


def vpn_subscription() -> str:
    """Return a Mozilla account subscription that provides the VPN"""
    assert settings.SUBSCRIPTIONS_WITH_VPN
    vpn_only_plans = list(
        set(settings.SUBSCRIPTIONS_WITH_VPN)
        - set(settings.SUBSCRIPTIONS_WITH_PHONE)
        - set(settings.SUBSCRIPTIONS_WITH_UNLIMITED)
    )
    assert vpn_only_plans
    return random.choice(vpn_only_plans)


def omit_markus_logs(caplog: pytest.LogCaptureFixture) -> list[LogRecord]:
    """
    Return log records that are not markus debug logs.

    Markus debug logs are enabled at Django setup with STATSD_DEBUG=True
    """
    return [rec for rec in caplog.records if rec.name != "markus"]


def log_extra(log_record: LogRecord) -> dict[str, Any]:
    """Reconstruct the "extra" argument to the log call"""
    omit_log_record_keys = {
        "args",
        "created",
        "exc_info",
        "exc_text",
        "filename",
        "funcName",
        "levelname",
        "levelno",
        "lineno",
        "message",
        "module",
        "msecs",
        "msg",
        "name",
        "pathname",
        "process",
        "processName",
        "relativeCreated",
        "rid",
        "stack_info",
        "taskName",
        "thread",
        "threadName",
    }
    return {
        key: val
        for key, val in log_record.__dict__.items()
        if key not in omit_log_record_keys
    }


def create_expected_glean_event(
    category: str,
    name: str,
    user: User,
    extra_items: dict[str, str],
    event_time: str,
) -> dict[str, str | dict[str, str]]:
    """
    Return the expected 'event' section of the event payload.

    category: The Glean event category
    name: The Glean event name / subcategory
    user: The requesting user. The fxa_id, date_joined_relay, date_joined_premium, and
      premium_status will be extracted from the user.
    extra_items: Additional or override extra items for this event
    event_time: The time of the event
    """
    user_extra_items: dict[str, str] = {}

    # Get values from the user object
    if user.profile.fxa:
        user_extra_items["fxa_id"] = user.profile.metrics_fxa_id
        user_extra_items["premium_status"] = user.profile.metrics_premium_status
    user_extra_items["date_joined_relay"] = str(int(user.date_joined.timestamp()))
    if user.profile.date_subscribed:
        user_extra_items["date_joined_premium"] = str(
            int(user.profile.date_subscribed.timestamp())
        )

    extra = (
        {
            "fxa_id": "",
            "platform": "",
            "n_random_masks": "0",
            "n_domain_masks": "0",
            "n_deleted_random_masks": "0",
            "n_deleted_domain_masks": "0",
            "date_joined_relay": "-1",
            "premium_status": "free",
            "date_joined_premium": "-1",
            "has_extension": "false",
            "date_got_extension": "-2",
        }
        | user_extra_items
        | extra_items
    )
    return {
        "category": category,
        "name": name,
        "extra": extra,
        "timestamp": event_time,
    }


def get_glean_event(
    caplog: pytest.LogCaptureFixture | _LoggingWatcher,
    category: str | None = None,
    name: str | None = None,
    exclude_api_access: bool = True,
) -> dict[str, Any] | None:
    """Return the event payload from a Glean server event log."""
    for record in caplog.records:
        if record.name == "glean-server-event":
            assert hasattr(record, "payload")
            event = json.loads(record.payload)["events"][0]
            assert isinstance(event, dict)
            if (
                exclude_api_access
                and event["category"] == "api"
                and event["name"] == "accessed"
            ):
                continue
            if (not category or event["category"] == category) and (
                not name or event["name"] == name
            ):
                return event
    return None

1	"""Helper functions for tests"""
2
3	import json	1✔
4	import random	1✔
5	from datetime import UTC, datetime	1✔
6	from logging import LogRecord	1✔
7	from typing import Any	1✔
8	from unittest._log import _LoggingWatcher	1✔
9	from uuid import uuid4	1✔
10
11	from django.conf import settings	1✔
12	from django.contrib.auth.models import User	1✔
13
14	import pytest	1✔
15	from allauth.socialaccount.models import SocialAccount	1✔
16	from model_bakery import baker	1✔
17
18
19	def make_free_test_user(email: str = "") -> User:	1✔
20	"""Make a user who has signed up for the free Relay plan."""
21	if email:	1✔
22	user = baker.make(User, email=email)	1✔
23	else:
24	user = baker.make(User)	1✔
25	user.profile.server_storage = True	1✔
26	user.profile.save()	1✔
27	uid = str(uuid4())	1✔
28	baker.make(	1✔
29	SocialAccount,
30	user=user,
31	uid=uid,
32	provider="fxa",
33	extra_data={"avatar": "avatar.png", "uid": uid},
34	)
35	return user	1✔
36
37
38	def make_premium_test_user() -> User:	1✔
39	"""Make a user who has the premium Relay plan, but hasn't picked a subdomain."""
40	premium_user = baker.make(User, email="premium@email.com")	1✔
41	premium_user.profile.server_storage = True	1✔
42	premium_user.profile.date_subscribed = datetime.now(tz=UTC)	1✔
43	premium_user.profile.save()	1✔
44	upgrade_test_user_to_premium(premium_user)	1✔
45	return premium_user	1✔
46
47
48	def upgrade_test_user_to_premium(user: User) -> None:	1✔
49	"""Create an FxA SocialAccount with an unlimited email masks plan."""
50	if SocialAccount.objects.filter(user=user).exists():	1!
51	raise Exception("upgrade_test_user_to_premium does not (yet) handle this.")	×
52	uid = str(uuid4())	1✔
53	baker.make(	1✔
54	SocialAccount,
55	user=user,
56	uid=uid,
57	provider="fxa",
58	extra_data={
59	"avatar": "avatar.png",
60	"subscriptions": [premium_subscription()],
61	"uid": uid,
62	},
63	)
64
65
66	def make_storageless_test_user() -> User:	1✔
67	storageless_user = baker.make(User)	1✔
68	storageless_user_profile = storageless_user.profile	1✔
69	storageless_user_profile.server_storage = False	1✔
70	storageless_user_profile.subdomain = "mydomain"	1✔
71	storageless_user_profile.date_subscribed = datetime.now(tz=UTC)	1✔
72	storageless_user_profile.save()	1✔
73	upgrade_test_user_to_premium(storageless_user)	1✔
74	return storageless_user	1✔
75
76
77	def premium_subscription() -> str:	1✔
78	"""Return a Mozilla account subscription that provides unlimited emails"""
79	assert settings.SUBSCRIPTIONS_WITH_UNLIMITED	1✔
80	premium_only_plans = list(	1✔
81	set(settings.SUBSCRIPTIONS_WITH_UNLIMITED)
82	- set(settings.SUBSCRIPTIONS_WITH_PHONE)
83	- set(settings.SUBSCRIPTIONS_WITH_VPN)
84	)
85	assert premium_only_plans	1✔
86	return random.choice(premium_only_plans)	1✔
87
88
89	def phone_subscription() -> str:	1✔
90	"""Return a Mozilla account subscription that provides a phone mask"""
91	assert settings.SUBSCRIPTIONS_WITH_PHONE	1✔
92	phones_only_plans = list(	1✔
93	set(settings.SUBSCRIPTIONS_WITH_PHONE)
94	- set(settings.SUBSCRIPTIONS_WITH_VPN)
95	- set(settings.SUBSCRIPTIONS_WITH_UNLIMITED)
96	)
97	assert phones_only_plans	1✔
98	return random.choice(phones_only_plans)	1✔
99
100
101	def vpn_subscription() -> str:	1✔
102	"""Return a Mozilla account subscription that provides the VPN"""
103	assert settings.SUBSCRIPTIONS_WITH_VPN	1✔
104	vpn_only_plans = list(	1✔
105	set(settings.SUBSCRIPTIONS_WITH_VPN)
106	- set(settings.SUBSCRIPTIONS_WITH_PHONE)
107	- set(settings.SUBSCRIPTIONS_WITH_UNLIMITED)
108	)
109	assert vpn_only_plans	1✔
110	return random.choice(vpn_only_plans)	1✔
111
112
113	def omit_markus_logs(caplog: pytest.LogCaptureFixture) -> list[LogRecord]:	1✔
114	"""
115	Return log records that are not markus debug logs.
116
117	Markus debug logs are enabled at Django setup with STATSD_DEBUG=True
118	"""
119	return [rec for rec in caplog.records if rec.name != "markus"]	1✔
120
121
122	def log_extra(log_record: LogRecord) -> dict[str, Any]:	1✔
123	"""Reconstruct the "extra" argument to the log call"""
124	omit_log_record_keys = {	1✔
125	"args",
126	"created",
127	"exc_info",
128	"exc_text",
129	"filename",
130	"funcName",
131	"levelname",
132	"levelno",
133	"lineno",
134	"message",
135	"module",
136	"msecs",
137	"msg",
138	"name",
139	"pathname",
140	"process",
141	"processName",
142	"relativeCreated",
143	"rid",
144	"stack_info",
145	"taskName",
146	"thread",
147	"threadName",
148	}
149	return {	1✔
150	key: val
151	for key, val in log_record.__dict__.items()
152	if key not in omit_log_record_keys
153	}
154
155
156	def create_expected_glean_event(	1✔
157	category: str,
158	name: str,
159	user: User,
160	extra_items: dict[str, str],
161	event_time: str,
162	) -> dict[str, str \| dict[str, str]]:
163	"""
164	Return the expected 'event' section of the event payload.
165
166	category: The Glean event category
167	name: The Glean event name / subcategory
168	user: The requesting user. The fxa_id, date_joined_relay, date_joined_premium, and
169	premium_status will be extracted from the user.
170	extra_items: Additional or override extra items for this event
171	event_time: The time of the event
172	"""
173	user_extra_items: dict[str, str] = {}	1✔
174
175	# Get values from the user object
176	if user.profile.fxa:	1✔
177	user_extra_items["fxa_id"] = user.profile.metrics_fxa_id	1✔
178	user_extra_items["premium_status"] = user.profile.metrics_premium_status	1✔
179	user_extra_items["date_joined_relay"] = str(int(user.date_joined.timestamp()))	1✔
180	if user.profile.date_subscribed:	1✔
181	user_extra_items["date_joined_premium"] = str(	1✔
182	int(user.profile.date_subscribed.timestamp())
183	)
184
185	extra = (	1✔
186	{
187	"fxa_id": "",
188	"platform": "",
189	"n_random_masks": "0",
190	"n_domain_masks": "0",
191	"n_deleted_random_masks": "0",
192	"n_deleted_domain_masks": "0",
193	"date_joined_relay": "-1",
194	"premium_status": "free",
195	"date_joined_premium": "-1",
196	"has_extension": "false",
197	"date_got_extension": "-2",
198	}
199	\| user_extra_items
200	\| extra_items
201	)
202	return {	1✔
203	"category": category,
204	"name": name,
205	"extra": extra,
206	"timestamp": event_time,
207	}
208
209
210	def get_glean_event(	1✔
211	caplog: pytest.LogCaptureFixture \| _LoggingWatcher,
212	category: str \| None = None,
213	name: str \| None = None,
214	exclude_api_access: bool = True,
215	) -> dict[str, Any] \| None:
216	"""Return the event payload from a Glean server event log."""
217	for record in caplog.records:	1✔
218	if record.name == "glean-server-event":	1✔
219	assert hasattr(record, "payload")	1✔
220	event = json.loads(record.payload)["events"][0]	1✔
221	assert isinstance(event, dict)	1✔
222	if (	1!
223	exclude_api_access
224	and event["category"] == "api"
225	and event["name"] == "accessed"
226	):
NEW 227	continue	×
228	if (not category or event["category"] == category) and (	1✔
229	not name or event["name"] == name
230	):
231	return event	1✔
232	return None	1✔

mozilla / fx-private-relay / 1f27d7ef-045c-4b13-be21-3a6fa9a7e5a6

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous