6e0eae52-0c36-4319-b852-15f966a6a5b0

Committed 16 May 2025 12:32PM UTC coverage: 85.235% (+0.03%) from 85.206%

Build # 6e0eae52-0c36-4319-b852-15f966a6a5b0

Build Type

Pull #5572

circleci

Committed by

groovecoder

Commit Message

for MPP-3439: add IDNAEmailCleaner to clean email address domains with non-ASCII chars

Pull Request Pull Request #5572: for MPP-3439: add IDNAEmailCleaner to clean email address domains with non-ASCII chars

Run Details

2464 of 3609 branches covered (68.27%)

Branch coverage included in aggregate %.

56 of 57 new or added lines in 2 files covered. (98.25%)

3 existing lines in 1 file now uncovered.

17372 of 19663 relevant lines covered (88.35%)

9.66 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

96.43

/privaterelay/cleaners.py

"""Tasks that detect and fix data issues in privaterelay app or 3rd party apps."""

from django.contrib.auth.models import User
from django.db.models import Q, QuerySet, Value
from django.db.models.functions import Coalesce, NullIf

from allauth.socialaccount.models import SocialAccount, SocialApp

from .cleaner_task import CleanerTask, DataBisectSpec, DataModelSpec


class MissingEmailCleaner(CleanerTask):
    slug = "missing-email"
    title = "Ensure all users have an email"
    check_description = (
        "When User.email is empty, we are unable to forward email to the Relay user."
        " We can get the email from the FxA profile if available."
    )

    # The Firefox Accounts default provider identifier is `fxa`. Firefox Accounts was
    # the name for Mozilla Accounts before 2023. This query returns the value, as a
    # one-element list, of the `SocialApp.provider_id` if set, and `fxa` if not.
    #
    # The `provider` field for a SocialAccount is a CharField, not a ForeignKey.  The
    # default `provider` value is the `id` of the SocialAccount provider. This `id` is
    # used in the django-allauth URLs. The `provider` value can be overridden by setting
    # the `SocialApp.provider_id`. This supports generic providers like OpenID Connect
    # and SAML. When it is set on a non-generic provider, it changes the value of the
    # SocialAccount's `provider`, but not the URLs. When django-allauth needs the
    # SocialApp for a given SocialAccount, it uses an adapter to look it up at runtime.
    #
    # See:
    # - The Firefox Account Provider docs
    # https://docs.allauth.org/en/latest/socialaccount/providers/fxa.html
    # - The OpenID Connect docs
    # https://docs.allauth.org/en/latest/socialaccount/providers/openid_connect.html
    # - The DefaultSocialAccountAdapter docs
    # https://docs.allauth.org/en/latest/socialaccount/adapter.html#allauth.socialaccount.adapter.DefaultSocialAccountAdapter.get_provider

    _fxa_provider_id = SocialApp.objects.filter(provider="fxa").values_list(
        Coalesce(NullIf("provider_id", Value("")), "provider"), flat=True
    )

    data_specification = [
        # Report on how many users do not have an email
        DataModelSpec(
            model=User,
            subdivisions=[
                DataBisectSpec("email", ~Q(email__exact="")),
                DataBisectSpec(
                    "!email.fxa", Q(socialaccount__provider__in=_fxa_provider_id)
                ),
            ],
            omit_key_prefixes=["!email.!fxa"],
            report_name_overrides={"!email": "No Email", "!email.fxa": "Has FxA"},
            ok_key="email",
            needs_cleaning_key="!email.fxa",
        )
    ]

    def clean_users(self, queryset: QuerySet[User]) -> int:
        fixed = 0
        for user in queryset:
            try:
                fxa_account = SocialAccount.objects.get(
                    provider__in=self._fxa_provider_id, user=user
                )
            except SocialAccount.DoesNotExist:
                continue
            if fxa_email := fxa_account.extra_data.get("email"):
                user.email = fxa_email
                user.save(update_fields=["email"])
                fixed += 1
        return fixed


class IDNAEmailCleaner(CleanerTask):
    slug = "idna-email"
    title = "Fix non-ASCII domains in user emails"
    check_description = (
        "Users with non-ASCII characters in their email domain cannot receive emails"
        "via AWS SES. Convert these domains to ASCII-compatible Punycode using IDNA."
    )

    def has_non_ascii_domain(self, email: str) -> bool:
        try:
            domain = email.split("@", 1)[1]
            domain.encode("ascii")
            return False
        except (IndexError, UnicodeEncodeError):
            return True

    def punycode_email(self, email: str) -> str:
        local, domain = email.split("@", 1)
        domain_ascii = domain.encode("idna").decode("ascii")
        return f"{local}@{domain_ascii}"

    data_specification = [
        DataModelSpec(
            model=User,
            subdivisions=[
                DataBisectSpec(
                    "ascii_domain",
                    ~Q(email__regex=r".*@.*[^\x00-\x7F].*"),
                )
            ],
            report_name_overrides={
                "ascii_domain": "Has ASCII Domain",
                "!ascii_domain": "Non-ASCII Domain",
            },
            ok_key="ascii_domain",
            needs_cleaning_key="!ascii_domain",
        )
    ]

    def clean_users(self, queryset: QuerySet[User]) -> int:
        fixed = 0
        for user in queryset:
            if user.email and self.has_non_ascii_domain(user.email):
                updated_email = self.punycode_email(user.email)
                user.email = updated_email
                user.save(update_fields=["email"])
                # TODO: should we update their email in their SocialAccount?
                fixed += 1
        return fixed

1	"""Tasks that detect and fix data issues in privaterelay app or 3rd party apps."""
2
3	from django.contrib.auth.models import User	1✔
4	from django.db.models import Q, QuerySet, Value	1✔
5	from django.db.models.functions import Coalesce, NullIf	1✔
6
7	from allauth.socialaccount.models import SocialAccount, SocialApp	1✔
8
9	from .cleaner_task import CleanerTask, DataBisectSpec, DataModelSpec	1✔
10
11
12	class MissingEmailCleaner(CleanerTask):	1✔
13	slug = "missing-email"	1✔
14	title = "Ensure all users have an email"	1✔
15	check_description = (	1✔
16	"When User.email is empty, we are unable to forward email to the Relay user."
17	" We can get the email from the FxA profile if available."
18	)
19
20	# The Firefox Accounts default provider identifier is `fxa`. Firefox Accounts was
21	# the name for Mozilla Accounts before 2023. This query returns the value, as a
22	# one-element list, of the `SocialApp.provider_id` if set, and `fxa` if not.
23	#
24	# The `provider` field for a SocialAccount is a CharField, not a ForeignKey. The
25	# default `provider` value is the `id` of the SocialAccount provider. This `id` is
26	# used in the django-allauth URLs. The `provider` value can be overridden by setting
27	# the `SocialApp.provider_id`. This supports generic providers like OpenID Connect
28	# and SAML. When it is set on a non-generic provider, it changes the value of the
29	# SocialAccount's `provider`, but not the URLs. When django-allauth needs the
30	# SocialApp for a given SocialAccount, it uses an adapter to look it up at runtime.
31	#
32	# See:
33	# - The Firefox Account Provider docs
34	# https://docs.allauth.org/en/latest/socialaccount/providers/fxa.html
35	# - The OpenID Connect docs
36	# https://docs.allauth.org/en/latest/socialaccount/providers/openid_connect.html
37	# - The DefaultSocialAccountAdapter docs
38	# https://docs.allauth.org/en/latest/socialaccount/adapter.html#allauth.socialaccount.adapter.DefaultSocialAccountAdapter.get_provider
39
40	_fxa_provider_id = SocialApp.objects.filter(provider="fxa").values_list(	1✔
41	Coalesce(NullIf("provider_id", Value("")), "provider"), flat=True
42	)
43
44	data_specification = [	1✔
45	# Report on how many users do not have an email
46	DataModelSpec(
47	model=User,
48	subdivisions=[
49	DataBisectSpec("email", ~Q(email__exact="")),
50	DataBisectSpec(
51	"!email.fxa", Q(socialaccount__provider__in=_fxa_provider_id)
52	),
53	],
54	omit_key_prefixes=["!email.!fxa"],
55	report_name_overrides={"!email": "No Email", "!email.fxa": "Has FxA"},
56	ok_key="email",
57	needs_cleaning_key="!email.fxa",
58	)
59	]
60
61	def clean_users(self, queryset: QuerySet[User]) -> int:	1✔
62	fixed = 0	1✔
63	for user in queryset:	1✔
64	try:	1✔
65	fxa_account = SocialAccount.objects.get(	1✔
66	provider__in=self._fxa_provider_id, user=user
67	)
68	except SocialAccount.DoesNotExist:	1✔
69	continue	1✔
70	if fxa_email := fxa_account.extra_data.get("email"):	1✔
71	user.email = fxa_email	1✔
72	user.save(update_fields=["email"])	1✔
73	fixed += 1	1✔
74	return fixed	1✔
75
76
77	class IDNAEmailCleaner(CleanerTask):	1✔
78	slug = "idna-email"	1✔
79	title = "Fix non-ASCII domains in user emails"	1✔
80	check_description = (	1✔
81	"Users with non-ASCII characters in their email domain cannot receive emails"
82	"via AWS SES. Convert these domains to ASCII-compatible Punycode using IDNA."
83	)
84
85	def has_non_ascii_domain(self, email: str) -> bool:	1✔
86	try:	1✔
87	domain = email.split("@", 1)[1]	1✔
88	domain.encode("ascii")	1✔
NEW 89	return False	×
90	except (IndexError, UnicodeEncodeError):	1✔
91	return True	1✔
92
93	def punycode_email(self, email: str) -> str:	1✔
94	local, domain = email.split("@", 1)	1✔
95	domain_ascii = domain.encode("idna").decode("ascii")	1✔
96	return f"{local}@{domain_ascii}"	1✔
97
98	data_specification = [	1✔
99	DataModelSpec(
100	model=User,
101	subdivisions=[
102	DataBisectSpec(
103	"ascii_domain",
104	~Q(email__regex=r".@.[^\x00-\x7F].*"),
105	)
106	],
107	report_name_overrides={
108	"ascii_domain": "Has ASCII Domain",
109	"!ascii_domain": "Non-ASCII Domain",
110	},
111	ok_key="ascii_domain",
112	needs_cleaning_key="!ascii_domain",
113	)
114	]
115
116	def clean_users(self, queryset: QuerySet[User]) -> int:	1✔
117	fixed = 0	1✔
118	for user in queryset:	1✔
119	if user.email and self.has_non_ascii_domain(user.email):	1!
120	updated_email = self.punycode_email(user.email)	1✔
121	user.email = updated_email	1✔
122	user.save(update_fields=["email"])	1✔
123	# TODO: should we update their email in their SocialAccount?
124	fixed += 1	1✔
125	return fixed	1✔

mozilla / fx-private-relay / 6e0eae52-0c36-4319-b852-15f966a6a5b0

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous