809b7f7d-bc91-48cd-9e16-1b3038ce95de

Committed 14 May 2025 01:36PM CUT coverage: 85.189% (-0.05%) from 85.235%

Build # 809b7f7d-bc91-48cd-9e16-1b3038ce95de

Build Type

Pull #5550

circleci

Committed by

groovecoder

Commit Message

for MPP-3957: start update_fxrelay_allowlist_collection command

Pull Request Pull Request #5550: for MPP-3957: start update_fxrelay_allowlist_collection command

Run Details

2468 of 3609 branches covered (68.38%)

Branch coverage included in aggregate %.

77 of 101 new or added lines in 3 files covered. (76.24%)

1 existing line in 1 file now uncovered.

17393 of 19705 relevant lines covered (88.27%)

9.64 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

68.6

/privaterelay/management/commands/update_fxrelay_allowlist_collection.py

from django.conf import settings
from django.core.management.base import BaseCommand

import requests
from kinto_http import Client, KintoException
from kinto_http.patch_type import BasicPatch

BUCKET = settings.KINTO_BUCKET
COLLECTION = settings.KINTO_COLLECTION


class Command(BaseCommand):
    help = "Updates the Firefox Relay allowlist Kinto collection from a JSON source."

    def handle(self, *args, **options):
        how_many_changes = 0

        print(f"📥 Loading new allowlist from {settings.ALLOWLIST_INPUT_URL}")
        response = requests.get(settings.ALLOWLIST_INPUT_URL, timeout=30)
        response.raise_for_status()
        new_allowlist = response.content.decode()
        new_domains = set(filter(None, new_allowlist.split("\n")))
        print(
            f"📋 Parsed {len(new_domains)} domains from {settings.ALLOWLIST_INPUT_URL}."
        )

        print(
            "📥 Getting existing domain records from "
            f"{settings.KINTO_SERVER}, "
            f"🪣 bucket {BUCKET}, 📁 collection {COLLECTION} ..."
        )
        print(f" ☁️ Connecting to {settings.KINTO_SERVER} ...")
        client = Client(
            server_url=settings.KINTO_SERVER, auth=settings.KINTO_AUTH_TOKEN
        )
        try:
            client.server_info()
            print(" ✅ Connected to Kinto server successfully.")
        except Exception as e:
            self.stderr.write(f" ❌ Failed to connect to Kinto server: {e}")
            return

        print(f" 🪣 Ensuring bucket {BUCKET} exists ...")
        try:
            client.get_bucket(id=BUCKET)
            print(f" ✅ Bucket {BUCKET} already exists.")
        except KintoException:
            print(f" ❓ Bucket {BUCKET} not found. Creating...")
            try:
                client.create_bucket(id=BUCKET)
                print(f" ✅ Bucket {BUCKET} created.")
            except KintoException as e:
                self.stderr.write(f" ❌ Failed to find or create bucket: {e}")
                return

        print(f" 📁 Ensuring collection {COLLECTION} exists ...")
        try:
            client.get_collection(id=COLLECTION, bucket=BUCKET)
            print(f" ✅ Collection {COLLECTION} already exists.")
        except KintoException:
            print(f" ❓ Collection {COLLECTION} not found. Creating...")
            try:
                client.create_collection(id=COLLECTION, bucket=BUCKET)
                print(f" ✅ Collection {COLLECTION} created.")
            except KintoException as e:
                self.stderr.write(f" ❌ Failed to find or create collection: {e}")
                return

        existing_records = client.get_records(bucket=BUCKET, collection=COLLECTION)
        existing_domains = {rec["domain"] for rec in existing_records}
        print(f"📋 Found {len(existing_domains)} existing domains.")

        # Delete records:
        # 1. no longer in the domain allowlist
        # 2. where id does not match domain
        print("🔎 Checking for changes ...")
        for existing_record in existing_records:
            if (
                existing_record["domain"] not in new_domains
                or existing_record["domain"].replace(".", "-") != existing_record["id"]
            ):
                print(f' 🗑 removed domain: {existing_record["domain"]}')
                client.delete_record(
                    id=existing_record["id"], bucket=BUCKET, collection=COLLECTION
                )
                how_many_changes += 1

        # Add new records for new domains in allowlist
        for domain in new_domains:
            if domain not in existing_domains:
                print(f" ✚ new domain: {domain}")
                record = {
                    "domain": domain,
                }
                client.create_record(
                    id=domain.replace(".", "-"),
                    data=record,
                    bucket=BUCKET,
                    collection=COLLECTION,
                )
                how_many_changes += 1

        if how_many_changes == 0:
            print(" ⚪️ No changes found.")
            print("✅ Done.")
            return

        print(" 📝 Made {how_many_changes} changes.")
        # Request review by updating the collection metadata
        try:
            print(f" 📤 Requesting review for collection {COLLECTION}.")
            client.patch_collection(
                id=COLLECTION,
                bucket=BUCKET,
                changes=BasicPatch({"status": "to-review"}),
            )
        except KintoException as e:
            self.stderr.write(f" ❌ Failed to request review: {e}")
        print("✅ Done.")

1	from django.conf import settings	1✔
2	from django.core.management.base import BaseCommand	1✔
3
4	import requests	1✔
5	from kinto_http import Client, KintoException	1✔
6	from kinto_http.patch_type import BasicPatch	1✔
7
8	BUCKET = settings.KINTO_BUCKET	1✔
9	COLLECTION = settings.KINTO_COLLECTION	1✔
10
11
12	class Command(BaseCommand):	1✔
13	help = "Updates the Firefox Relay allowlist Kinto collection from a JSON source."	1✔
14
15	def handle(self, args, *options):	1✔
16	how_many_changes = 0	1✔
17
18	print(f"📥 Loading new allowlist from {settings.ALLOWLIST_INPUT_URL}")	1✔
19	response = requests.get(settings.ALLOWLIST_INPUT_URL, timeout=30)	1✔
20	response.raise_for_status()	1✔
21	new_allowlist = response.content.decode()	1✔
22	new_domains = set(filter(None, new_allowlist.split("\n")))	1✔
23	print(	1✔
24	f"📋 Parsed {len(new_domains)} domains from {settings.ALLOWLIST_INPUT_URL}."
25	)
26
27	print(	1✔
28	"📥 Getting existing domain records from "
29	f"{settings.KINTO_SERVER}, "
30	f"🪣 bucket {BUCKET}, 📁 collection {COLLECTION} ..."
31	)
32	print(f" ☁️ Connecting to {settings.KINTO_SERVER} ...")	1✔
33	client = Client(	1✔
34	server_url=settings.KINTO_SERVER, auth=settings.KINTO_AUTH_TOKEN
35	)
36	try:	1✔
37	client.server_info()	1✔
38	print(" ✅ Connected to Kinto server successfully.")	1✔
NEW 39	except Exception as e:	×
NEW 40	self.stderr.write(f" ❌ Failed to connect to Kinto server: {e}")	×
NEW 41	return	×
42
43	print(f" 🪣 Ensuring bucket {BUCKET} exists ...")	1✔
44	try:	1✔
45	client.get_bucket(id=BUCKET)	1✔
46	print(f" ✅ Bucket {BUCKET} already exists.")	1✔
NEW 47	except KintoException:	×
NEW 48	print(f" ❓ Bucket {BUCKET} not found. Creating...")	×
NEW 49	try:	×
NEW 50	client.create_bucket(id=BUCKET)	×
NEW 51	print(f" ✅ Bucket {BUCKET} created.")	×
NEW 52	except KintoException as e:	×
NEW 53	self.stderr.write(f" ❌ Failed to find or create bucket: {e}")	×
NEW 54	return	×
55
56	print(f" 📁 Ensuring collection {COLLECTION} exists ...")	1✔
57	try:	1✔
58	client.get_collection(id=COLLECTION, bucket=BUCKET)	1✔
59	print(f" ✅ Collection {COLLECTION} already exists.")	1✔
NEW 60	except KintoException:	×
NEW 61	print(f" ❓ Collection {COLLECTION} not found. Creating...")	×
NEW 62	try:	×
NEW 63	client.create_collection(id=COLLECTION, bucket=BUCKET)	×
NEW 64	print(f" ✅ Collection {COLLECTION} created.")	×
NEW 65	except KintoException as e:	×
NEW 66	self.stderr.write(f" ❌ Failed to find or create collection: {e}")	×
NEW 67	return	×
68
69	existing_records = client.get_records(bucket=BUCKET, collection=COLLECTION)	1✔
70	existing_domains = {rec["domain"] for rec in existing_records}	1✔
71	print(f"📋 Found {len(existing_domains)} existing domains.")	1✔
72
73	# Delete records:
74	# 1. no longer in the domain allowlist
75	# 2. where id does not match domain
76	print("🔎 Checking for changes ...")	1✔
77	for existing_record in existing_records:	1✔
78	if (	1!
79	existing_record["domain"] not in new_domains
80	or existing_record["domain"].replace(".", "-") != existing_record["id"]
81	):
82	print(f' 🗑 removed domain: {existing_record["domain"]}')	1✔
83	client.delete_record(	1✔
84	id=existing_record["id"], bucket=BUCKET, collection=COLLECTION
85	)
86	how_many_changes += 1	1✔
87
88	# Add new records for new domains in allowlist
89	for domain in new_domains:	1✔
90	if domain not in existing_domains:	1!
91	print(f" ✚ new domain: {domain}")	1✔
92	record = {	1✔
93	"domain": domain,
94	}
95	client.create_record(	1✔
96	id=domain.replace(".", "-"),
97	data=record,
98	bucket=BUCKET,
99	collection=COLLECTION,
100	)
101	how_many_changes += 1	1✔
102
103	if how_many_changes == 0:	1!
NEW 104	print(" ⚪️ No changes found.")	×
NEW 105	print("✅ Done.")	×
NEW 106	return	×
107
108	print(" 📝 Made {how_many_changes} changes.")	1✔
109	# Request review by updating the collection metadata
110	try:	1✔
111	print(f" 📤 Requesting review for collection {COLLECTION}.")	1✔
112	client.patch_collection(	1✔
113	id=COLLECTION,
114	bucket=BUCKET,
115	changes=BasicPatch({"status": "to-review"}),
116	)
NEW 117	except KintoException as e:	×
NEW 118	self.stderr.write(f" ❌ Failed to request review: {e}")	×
119	print("✅ Done.")	1✔

mozilla / fx-private-relay / 809b7f7d-bc91-48cd-9e16-1b3038ce95de

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous