14844429278

Committed 05 May 2025 07:21PM UTC coverage: 78.703% (+1.3%) from 77.382%

Build # 14844429278

Build Type

push

github

Committed by

mcarans

Commit Message

Iterate over files repeatedly so as not to run out of memory

Run Details

144 of 158 new or added lines in 6 files covered. (91.14%)

2 existing lines in 1 file now uncovered.

728 of 925 relevant lines covered (78.7%)

0.79 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

27.66

/src/hdx/scraper/wfp/foodprices/world/__main__.py

#!/usr/bin/python
"""
Top level script. Calls other functions that generate datasets that this script then creates in HDX.

"""

import logging
import sys
from os.path import expanduser, join

from hdx.api.configuration import Configuration
from hdx.api.utilities.hdx_error_handler import HDXErrorHandler
from hdx.data.user import User
from hdx.facades.infer_arguments import facade
from hdx.location.wfp_api import WFPAPI
from hdx.scraper.wfp.foodprices._version import __version__
from hdx.scraper.wfp.foodprices.utilities import get_currencies, get_now
from hdx.scraper.wfp.foodprices.wfp_mappings import WFPMappings
from hdx.scraper.wfp.foodprices.world.dataset_generator import DatasetGenerator
from hdx.scraper.wfp.foodprices.world.global_markets import get_markets
from hdx.scraper.wfp.foodprices.world.global_prices_generator import (
    GlobalPricesGenerator,
)
from hdx.scraper.wfp.foodprices.world.hapi_dataset_generator import HAPIDatasetGenerator
from hdx.scraper.wfp.foodprices.world.hapi_output import HAPIOutput
from hdx.utilities.downloader import Download
from hdx.utilities.easy_logging import setup_logging
from hdx.utilities.loader import load_yaml
from hdx.utilities.path import (
    script_dir_plus_file,
    temp_dir_batch,
)
from hdx.utilities.retriever import Retrieve

setup_logging()
logger = logging.getLogger(__name__)

lookup = "hdx-scraper-wfp-foodprices"
updated_by_script = "HDX Scraper: WFP Food Prices"


def main(
    save: bool = False,
    use_saved: bool = False,
    countryiso3s: str = "",
    err_to_hdx: bool = False,
) -> None:
    """Generate datasets and create them in HDX

    Args:
        save (bool): Save all downloaded data. Defaults to False.
        use_saved (bool): Use saved data. Defaults to False.
        countryiso3s (str): Whether to limit to specific countries. Defaults to not limiting ("").
        err_to_hdx (bool): Whether to write any errors to HDX metadata. Defaults to False.

    Returns:
        None
    """
    logger.info(f"##### {lookup} version {__version__} ####")
    if not User.check_current_user_organization_access(
        "3ecac442-7fed-448d-8f78-b385ef6f84e7", "create_dataset"
    ):
        raise PermissionError("API Token does not give access to WFP organisation!")
    with HDXErrorHandler(write_to_hdx=err_to_hdx) as error_handler:
        with Download(
            fail_on_missing_file=False,
            extra_params_yaml=join(expanduser("~"), ".extraparams.yaml"),
            extra_params_lookup=lookup,
        ) as token_downloader:
            with Download(
                use_env=False, rate_limit={"calls": 1, "period": 0.1}
            ) as downloader:
                with temp_dir_batch(
                    lookup,
                    delete_if_exists=False,
                    delete_on_success=True,
                    delete_on_failure=False,
                ) as info:
                    if countryiso3s:
                        countryiso3s = countryiso3s.split(",")
                    else:
                        countryiso3s = None
                    folder = info["folder"]
                    batch = info["batch"]
                    retriever = Retrieve(
                        downloader, folder, "saved_data", folder, save, use_saved
                    )
                    configuration = Configuration.read()
                    base_configuration = script_dir_plus_file(
                        join("config", "project_configuration.yaml"), get_now
                    )
                    configuration.update(load_yaml(base_configuration))
                    wfp_api = WFPAPI(token_downloader, retriever)
                    wfp_api.update_retry_params(attempts=5, wait=3600)
                    wfp_mapping = WFPMappings(configuration, wfp_api, retriever)
                    _, commodities = wfp_mapping.build_commodity_category_mapping()
                    currencies = get_currencies(wfp_api)
                    markets = get_markets(downloader, folder)
                    if not markets:
                        logger.error("No markets data found!")
                        sys.exit(1)
                    prices_generator = GlobalPricesGenerator(
                        configuration, downloader, folder
                    )
                    start_date, end_date = prices_generator.get_years_per_country()
                    year_to_pricespath = prices_generator.create_prices_files()
                    if not year_to_pricespath:
                        logger.error("No prices data found!")
                        sys.exit(1)
                    dataset_generator = DatasetGenerator(
                        configuration, folder, start_date, end_date
                    )
                    dataset, showcase = (
                        dataset_generator.generate_global_dataset_and_showcase(
                            year_to_pricespath, markets, commodities, currencies
                        )
                    )
                    snippet = "Countries, Commodities and Markets data"
                    dataset.update_from_yaml(
                        script_dir_plus_file(
                            join("config", "hdx_dataset_static.yaml"), get_now
                        )
                    )
                    dataset["notes"] = dataset["notes"] % snippet
                    dataset.preview_off()
                    dataset.create_in_hdx(
                        remove_additional_resources=True,
                        match_resource_order=True,
                        hxl_update=False,
                        updated_by_script=updated_by_script,
                        batch=batch,
                    )
                    showcase.create_in_hdx()
                    showcase.add_dataset(dataset)

                    year_to_prices_resource_id = {}
                    markets_resource_id = None
                    for resource in dataset.get_resources():
                        resource_name = resource["name"]
                        if dataset_generator.global_prices_name in resource_name:
                            year = int(resource_name[-4:])
                            year_to_prices_resource_id[year] = resource["id"]
                        elif resource_name == dataset_generator.global_markets_name:
                            markets_resource_id = resource["id"]
                        elif resource_name == dataset_generator.global_commodities_name:
                            commodities_resource_id = resource["id"]
                        elif resource_name == dataset_generator.global_currencies_name:
                            currencies_resource_id = resource["id"]
                    if year_to_prices_resource_id and markets_resource_id:
                        dataset_id = dataset["id"]
                        hapi_output = HAPIOutput(
                            configuration,
                            downloader,
                            folder,
                            error_handler,
                        )
                        hapi_output.setup_admins(retriever, countryiso3s)
                        hapi_currencies = hapi_output.process_currencies(
                            currencies, dataset_id, currencies_resource_id
                        )
                        hapi_commodities = hapi_output.process_commodities(
                            commodities,
                            dataset_id,
                            commodities_resource_id,
                        )
                        hapi_markets = hapi_output.process_markets(
                            markets, dataset_id, markets_resource_id
                        )
                        hapi_year_to_pricespath = hapi_output.create_prices_files(
                            year_to_pricespath, dataset_id, year_to_prices_resource_id
                        )
                        hapi_dataset_generator = HAPIDatasetGenerator(
                            configuration,
                            folder,
                            start_date,
                            end_date,
                        )
                        dataset = hapi_dataset_generator.generate_prices_dataset(
                            hapi_year_to_pricespath,
                            hapi_markets,
                            hapi_commodities,
                            hapi_currencies,
                        )
                        if dataset:
                            dataset.update_from_yaml(
                                script_dir_plus_file(
                                    join(
                                        "config",
                                        "hdx_hapi_dataset_static.yaml",
                                    ),
                                    main,
                                )
                            )
                            dataset.preview_off()
                            dataset.create_in_hdx(
                                remove_additional_resources=True,
                                match_resource_order=True,
                                hxl_update=False,
                                updated_by_script=updated_by_script,
                                batch=batch,
                            )
                            logger.info("WFP global HAPI dataset created")


if __name__ == "__main__":
    facade(
        main,
        user_agent_config_yaml=join(expanduser("~"), ".useragents.yaml"),
        user_agent_lookup=lookup,
        project_config_yaml=script_dir_plus_file(
            join("config", "project_configuration.yaml"), main
        ),
    )

1	#!/usr/bin/python
2	"""
3	Top level script. Calls other functions that generate datasets that this script then creates in HDX.
4
5	"""
6
7	import logging	1✔
8	import sys	1✔
9	from os.path import expanduser, join	1✔
10
11	from hdx.api.configuration import Configuration	1✔
12	from hdx.api.utilities.hdx_error_handler import HDXErrorHandler	1✔
13	from hdx.data.user import User	1✔
14	from hdx.facades.infer_arguments import facade	1✔
15	from hdx.location.wfp_api import WFPAPI	1✔
16	from hdx.scraper.wfp.foodprices._version import __version__	1✔
17	from hdx.scraper.wfp.foodprices.utilities import get_currencies, get_now	1✔
18	from hdx.scraper.wfp.foodprices.wfp_mappings import WFPMappings	1✔
19	from hdx.scraper.wfp.foodprices.world.dataset_generator import DatasetGenerator	1✔
20	from hdx.scraper.wfp.foodprices.world.global_markets import get_markets	1✔
21	from hdx.scraper.wfp.foodprices.world.global_prices_generator import (	1✔
22	GlobalPricesGenerator,
23	)
24	from hdx.scraper.wfp.foodprices.world.hapi_dataset_generator import HAPIDatasetGenerator	1✔
25	from hdx.scraper.wfp.foodprices.world.hapi_output import HAPIOutput	1✔
26	from hdx.utilities.downloader import Download	1✔
27	from hdx.utilities.easy_logging import setup_logging	1✔
28	from hdx.utilities.loader import load_yaml	1✔
29	from hdx.utilities.path import (	1✔
30	script_dir_plus_file,
31	temp_dir_batch,
32	)
33	from hdx.utilities.retriever import Retrieve	1✔
34
35	setup_logging()	1✔
36	logger = logging.getLogger(__name__)	1✔
37
38	lookup = "hdx-scraper-wfp-foodprices"	1✔
39	updated_by_script = "HDX Scraper: WFP Food Prices"	1✔
40
41
42	def main(	1✔
43	save: bool = False,
44	use_saved: bool = False,
45	countryiso3s: str = "",
46	err_to_hdx: bool = False,
47	) -> None:
48	"""Generate datasets and create them in HDX
49
50	Args:
51	save (bool): Save all downloaded data. Defaults to False.
52	use_saved (bool): Use saved data. Defaults to False.
53	countryiso3s (str): Whether to limit to specific countries. Defaults to not limiting ("").
54	err_to_hdx (bool): Whether to write any errors to HDX metadata. Defaults to False.
55
56	Returns:
57	None
58	"""
59	logger.info(f"##### {lookup} version {__version__} ####")	×
60	if not User.check_current_user_organization_access(	×
61	"3ecac442-7fed-448d-8f78-b385ef6f84e7", "create_dataset"
62	):
63	raise PermissionError("API Token does not give access to WFP organisation!")	×
64	with HDXErrorHandler(write_to_hdx=err_to_hdx) as error_handler:	×
65	with Download(	×
66	fail_on_missing_file=False,
67	extra_params_yaml=join(expanduser("~"), ".extraparams.yaml"),
68	extra_params_lookup=lookup,
69	) as token_downloader:
70	with Download(	×
71	use_env=False, rate_limit={"calls": 1, "period": 0.1}
72	) as downloader:
73	with temp_dir_batch(	×
74	lookup,
75	delete_if_exists=False,
76	delete_on_success=True,
77	delete_on_failure=False,
78	) as info:
79	if countryiso3s:	×
80	countryiso3s = countryiso3s.split(",")	×
81	else:
82	countryiso3s = None	×
83	folder = info["folder"]	×
84	batch = info["batch"]	×
85	retriever = Retrieve(	×
86	downloader, folder, "saved_data", folder, save, use_saved
87	)
88	configuration = Configuration.read()	×
89	base_configuration = script_dir_plus_file(	×
90	join("config", "project_configuration.yaml"), get_now
91	)
92	configuration.update(load_yaml(base_configuration))	×
93	wfp_api = WFPAPI(token_downloader, retriever)	×
94	wfp_api.update_retry_params(attempts=5, wait=3600)	×
95	wfp_mapping = WFPMappings(configuration, wfp_api, retriever)	×
96	_, commodities = wfp_mapping.build_commodity_category_mapping()	×
97	currencies = get_currencies(wfp_api)	×
NEW 98	markets = get_markets(downloader, folder)	×
UNCOV 99	if not markets:	×
100	logger.error("No markets data found!")	×
101	sys.exit(1)	×
NEW 102	prices_generator = GlobalPricesGenerator(	×
103	configuration, downloader, folder
104	)
NEW 105	start_date, end_date = prices_generator.get_years_per_country()	×
NEW 106	year_to_pricespath = prices_generator.create_prices_files()	×
NEW 107	if not year_to_pricespath:	×
NEW 108	logger.error("No prices data found!")	×
NEW 109	sys.exit(1)	×
NEW 110	dataset_generator = DatasetGenerator(	×
111	configuration, folder, start_date, end_date
112	)
UNCOV 113	dataset, showcase = (	×
114	dataset_generator.generate_global_dataset_and_showcase(
115	year_to_pricespath, markets, commodities, currencies
116	)
117	)
118	snippet = "Countries, Commodities and Markets data"	×
119	dataset.update_from_yaml(	×
120	script_dir_plus_file(
121	join("config", "hdx_dataset_static.yaml"), get_now
122	)
123	)
124	dataset["notes"] = dataset["notes"] % snippet	×
125	dataset.preview_off()	×
126	dataset.create_in_hdx(	×
127	remove_additional_resources=True,
128	match_resource_order=True,
129	hxl_update=False,
130	updated_by_script=updated_by_script,
131	batch=batch,
132	)
133	showcase.create_in_hdx()	×
134	showcase.add_dataset(dataset)	×
135
136	year_to_prices_resource_id = {}	×
137	markets_resource_id = None	×
138	for resource in dataset.get_resources():	×
139	resource_name = resource["name"]	×
140	if dataset_generator.global_prices_name in resource_name:	×
141	year = int(resource_name[-4:])	×
142	year_to_prices_resource_id[year] = resource["id"]	×
143	elif resource_name == dataset_generator.global_markets_name:	×
144	markets_resource_id = resource["id"]	×
145	elif resource_name == dataset_generator.global_commodities_name:	×
146	commodities_resource_id = resource["id"]	×
147	elif resource_name == dataset_generator.global_currencies_name:	×
148	currencies_resource_id = resource["id"]	×
149	if year_to_prices_resource_id and markets_resource_id:	×
150	dataset_id = dataset["id"]	×
151	hapi_output = HAPIOutput(	×
152	configuration,
153	downloader,
154	folder,
155	error_handler,
156	)
157	hapi_output.setup_admins(retriever, countryiso3s)	×
158	hapi_currencies = hapi_output.process_currencies(	×
159	currencies, dataset_id, currencies_resource_id
160	)
161	hapi_commodities = hapi_output.process_commodities(	×
162	commodities,
163	dataset_id,
164	commodities_resource_id,
165	)
166	hapi_markets = hapi_output.process_markets(	×
167	markets, dataset_id, markets_resource_id
168	)
NEW 169	hapi_year_to_pricespath = hapi_output.create_prices_files(	×
170	year_to_pricespath, dataset_id, year_to_prices_resource_id
171	)
172	hapi_dataset_generator = HAPIDatasetGenerator(	×
173	configuration,
174	folder,
175	start_date,
176	end_date,
177	)
178	dataset = hapi_dataset_generator.generate_prices_dataset(	×
179	hapi_year_to_pricespath,
180	hapi_markets,
181	hapi_commodities,
182	hapi_currencies,
183	)
184	if dataset:	×
185	dataset.update_from_yaml(	×
186	script_dir_plus_file(
187	join(
188	"config",
189	"hdx_hapi_dataset_static.yaml",
190	),
191	main,
192	)
193	)
194	dataset.preview_off()	×
195	dataset.create_in_hdx(	×
196	remove_additional_resources=True,
197	match_resource_order=True,
198	hxl_update=False,
199	updated_by_script=updated_by_script,
200	batch=batch,
201	)
202	logger.info("WFP global HAPI dataset created")	×
203
204
205	if __name__ == "__main__":
206	facade(
207	main,
208	user_agent_config_yaml=join(expanduser("~"), ".useragents.yaml"),
209	user_agent_lookup=lookup,
210	project_config_yaml=script_dir_plus_file(
211	join("config", "project_configuration.yaml"), main
212	),
213	)

OCHA-DAP / hdx-scraper-wfp-foodprices / 14844429278

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous