14117075234

Committed 27 Mar 2025 09:30PM UTC coverage: 77.988% (-0.6%) from 78.563%

Build # 14117075234

Build Type

push

github

Committed by

mcarans

Commit Message

Add garbage collection to see if it helps

Run Details

3 of 10 new or added lines in 1 file covered. (30.0%)

1 existing line in 1 file now uncovered.

659 of 845 relevant lines covered (77.99%)

0.78 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

24.41

/src/hdx/scraper/wfp/foodprices/__main__.py

#!/usr/bin/python
"""
Top level script. Calls other functions that generate datasets that this script then creates in HDX.

"""

import gc
import logging
from os.path import expanduser, join
from typing import Dict

import hdx.location.int_timestamp
from hdx.api.configuration import Configuration
from hdx.api.utilities.hdx_error_handler import HDXErrorHandler
from hdx.data.user import User
from hdx.database import Database
from hdx.facades.infer_arguments import facade
from hdx.location.currency import Currency
from hdx.location.wfp_api import WFPAPI
from hdx.scraper.wfp.foodprices._version import __version__
from hdx.scraper.wfp.foodprices.dataset_generator import DatasetGenerator
from hdx.scraper.wfp.foodprices.db_updater import DBUpdater
from hdx.scraper.wfp.foodprices.global_prices import get_global_prices_rows
from hdx.scraper.wfp.foodprices.hapi_dataset_generator import HAPIDatasetGenerator
from hdx.scraper.wfp.foodprices.hapi_output import HAPIOutput
from hdx.scraper.wfp.foodprices.utilities import get_now, setup_currency
from hdx.scraper.wfp.foodprices.wfp_food import WFPFood
from hdx.scraper.wfp.foodprices.wfp_mappings import WFPMappings
from hdx.utilities.downloader import Download
from hdx.utilities.easy_logging import setup_logging
from hdx.utilities.path import (
    progress_storing_folder,
    script_dir_plus_file,
    wheretostart_tempdir_batch,
)
from hdx.utilities.retriever import Retrieve

setup_logging()
logger = logging.getLogger(__name__)

lookup = "hdx-scraper-wfp-foodprices"
updated_by_script = "HDX Scraper: WFP Food Prices"


def main(
    save: bool = False,
    countryiso3s: str = "",
    use_saved: bool = False,
    save_wfp_rates: bool = True,
    err_to_hdx: bool = False,
) -> None:
    """Generate datasets and create them in HDX

    Args:
        save (bool): Save all downloaded data. Defaults to False.
        countryiso3s (str): Whether to limit to specific countries. Defaults to not limiting ("").
        use_saved (bool): Use saved data. Defaults to False.
        save_wfp_rates (bool): Save WFP FX rates data. Defaults to True.
        err_to_hdx (bool): Whether to write any errors to HDX metadata. Defaults to False.

    Returns:
        None
    """
    logger.info(f"##### {lookup} version {__version__} ####")
    if not User.check_current_user_organization_access(
        "3ecac442-7fed-448d-8f78-b385ef6f84e7", "create_dataset"
    ):
        raise PermissionError("API Token does not give access to WFP organisation!")
    with HDXErrorHandler(write_to_hdx=err_to_hdx) as error_handler:
        with Download(
            fail_on_missing_file=False,
            extra_params_yaml=join(expanduser("~"), ".extraparams.yaml"),
            extra_params_lookup=lookup,
        ) as token_downloader:
            with Download(
                use_env=False, rate_limit={"calls": 1, "period": 0.1}
            ) as downloader:
                with wheretostart_tempdir_batch(lookup) as info:
                    if countryiso3s:
                        countryiso3s = countryiso3s.split(",")
                    else:
                        countryiso3s = None
                    folder = info["folder"]
                    batch = info["batch"]
                    retriever = Retrieve(
                        downloader, folder, "saved_data", folder, save, use_saved
                    )
                    params = {
                        "dialect": "sqlite",
                        "database": f"/{folder}/foodprices.sqlite",
                    }
                    with Database(**params) as database:
                        configuration = Configuration.read()
                        now = get_now(retriever)
                        wfp_api = WFPAPI(token_downloader, retriever)
                        wfp_api.update_retry_params(attempts=5, wait=3600)
                        wfp = WFPMappings(configuration, wfp_api, retriever)
                        iso3_to_showcase_url = wfp.read_region_mapping()
                        iso3_to_source = wfp.read_source_overrides()
                        countries = wfp.get_countries(countryiso3s)
                        logger.info(
                            f"Number of country datasets to upload: {len(countries)}"
                        )
                        commodity_to_category, dbcommodities = (
                            wfp.build_commodity_category_mapping()
                        )
                        if save_wfp_rates:
                            wfp_rates_folder = folder
                        else:
                            wfp_rates_folder = None
                        currencies = setup_currency(
                            now, retriever, wfp_api, wfp_rates_folder
                        )
                        dataset_generator = DatasetGenerator(
                            configuration,
                            folder,
                            iso3_to_showcase_url,
                            iso3_to_source,
                            currencies,
                        )
                        dbupdater = DBUpdater(configuration, database)
                        dbupdater.update_commodities(dbcommodities)

                        def process_country(country: Dict[str, str]) -> None:
                            countryiso3 = country["iso3"]
                            dataset, showcase = (
                                dataset_generator.get_dataset_and_showcase(countryiso3)
                            )
                            if not dataset:
                                return
                            wfp_food = WFPFood(
                                countryiso3,
                                configuration,
                                iso3_to_showcase_url.get(countryiso3),
                                iso3_to_source.get(countryiso3),
                                commodity_to_category,
                            )
                            dbmarkets = wfp_food.get_price_markets(wfp_api)
                            if not dbmarkets:
                                return
                            rows, markets, sources = wfp_food.generate_rows(dbmarkets)
                            dataset, qc_indicators = dataset_generator.complete_dataset(
                                countryiso3, dataset, rows, markets, sources
                            )
                            time_period = dataset.get_time_period()
                            hdx_url = dataset.get_hdx_url()
                            dbupdater.update_tables(
                                countryiso3,
                                time_period,
                                hdx_url,
                                dbmarkets,
                            )

                            snippet = f"Food Prices data for {country['name']}"
                            if dataset:
                                dataset.update_from_yaml(
                                    script_dir_plus_file(
                                        join("config", "hdx_dataset_static.yaml"),
                                        main,
                                    )
                                )
                                dataset["notes"] = dataset["notes"] % (snippet, "")
                                dataset.generate_quickcharts(
                                    -1, indicators=qc_indicators
                                )
                                dataset.create_in_hdx(
                                    remove_additional_resources=True,
                                    hxl_update=False,
                                    updated_by_script=updated_by_script,
                                    batch=batch,
                                )
                                if showcase:
                                    showcase.create_in_hdx()
                                    showcase.add_dataset(dataset)
                                else:
                                    logger.info(
                                        f"{country['name']} does not have a showcase!"
                                    )

                        for _, country in progress_storing_folder(
                            info, countries, "iso3"
                        ):
                            process_country(country)

                        del hdx.location.int_timestamp._cache_timestamp_lookup
                        del Currency._cached_current_rates
                        del Currency._cached_historic_rates
                        del Currency._secondary_rates
                        del Currency._secondary_historic_rates
                        gc.collect()

                        table_data, start_date, end_date = (
                            dbupdater.get_data_from_tables()
                        )
                        global_prices_info = get_global_prices_rows(downloader, folder)
                        dataset, showcase = (
                            dataset_generator.generate_global_dataset_and_showcase(
                                global_prices_info, table_data, start_date, end_date
                            )
                        )
                        snippet = "Countries, Commodities and Markets data"
                        snippet2 = "The volume of data means that the actual Food Prices data is in country level datasets. "
                        dataset.update_from_yaml(
                            script_dir_plus_file(
                                join("config", "hdx_dataset_static.yaml"), main
                            )
                        )
                        dataset["notes"] = dataset["notes"] % (snippet, snippet2)
                        dataset.create_in_hdx(
                            remove_additional_resources=True,
                            hxl_update=False,
                            updated_by_script=updated_by_script,
                            batch=batch,
                        )
                        showcase.create_in_hdx()
                        showcase.add_dataset(dataset)

                        prices_resource_id = None
                        markets_resource_id = None
                        for resource in dataset.get_resources():
                            resource_name = resource["name"]
                            if resource_name == dataset_generator.global_name:
                                prices_resource_id = resource["id"]
                            elif resource_name == dataset_generator.global_markets_name:
                                markets_resource_id = resource["id"]
                        if prices_resource_id and markets_resource_id:
                            dataset_id = dataset["id"]
                            hapi_output = HAPIOutput(
                                configuration,
                                error_handler,
                            )
                            hapi_output.setup_admins(retriever, countryiso3s)
                            global_markets_info = table_data["DBMarket"]
                            hapi_output.process_markets(
                                global_markets_info, dataset_id, markets_resource_id
                            )
                            hapi_output.process_prices(
                                global_prices_info, dataset_id, prices_resource_id
                            )
                            hapi_dataset_generator = HAPIDatasetGenerator(
                                configuration,
                                global_markets_info,
                                global_prices_info,
                            )
                            dataset = hapi_dataset_generator.generate_prices_dataset(
                                folder,
                            )
                            if dataset:
                                dataset.update_from_yaml(
                                    script_dir_plus_file(
                                        join(
                                            "config",
                                            "hdx_hapi_dataset_static.yaml",
                                        ),
                                        main,
                                    )
                                )
                                dataset.create_in_hdx(
                                    remove_additional_resources=False,
                                    hxl_update=False,
                                    updated_by_script=updated_by_script,
                                    batch=batch,
                                )
                                logger.info("WFP global HAPI dataset created")


if __name__ == "__main__":
    facade(
        main,
        user_agent_config_yaml=join(expanduser("~"), ".useragents.yaml"),
        user_agent_lookup=lookup,
        project_config_yaml=script_dir_plus_file(
            join("config", "project_configuration.yaml"), main
        ),
    )

1	#!/usr/bin/python
2	"""
3	Top level script. Calls other functions that generate datasets that this script then creates in HDX.
4
5	"""
6
7	import gc	1✔
8	import logging	1✔
9	from os.path import expanduser, join	1✔
10	from typing import Dict	1✔
11
12	import hdx.location.int_timestamp	1✔
13	from hdx.api.configuration import Configuration	1✔
14	from hdx.api.utilities.hdx_error_handler import HDXErrorHandler	1✔
15	from hdx.data.user import User	1✔
16	from hdx.database import Database	1✔
17	from hdx.facades.infer_arguments import facade	1✔
18	from hdx.location.currency import Currency	1✔
19	from hdx.location.wfp_api import WFPAPI	1✔
20	from hdx.scraper.wfp.foodprices._version import __version__	1✔
21	from hdx.scraper.wfp.foodprices.dataset_generator import DatasetGenerator	1✔
22	from hdx.scraper.wfp.foodprices.db_updater import DBUpdater	1✔
23	from hdx.scraper.wfp.foodprices.global_prices import get_global_prices_rows	1✔
24	from hdx.scraper.wfp.foodprices.hapi_dataset_generator import HAPIDatasetGenerator	1✔
25	from hdx.scraper.wfp.foodprices.hapi_output import HAPIOutput	1✔
26	from hdx.scraper.wfp.foodprices.utilities import get_now, setup_currency	1✔
27	from hdx.scraper.wfp.foodprices.wfp_food import WFPFood	1✔
28	from hdx.scraper.wfp.foodprices.wfp_mappings import WFPMappings	1✔
29	from hdx.utilities.downloader import Download	1✔
30	from hdx.utilities.easy_logging import setup_logging	1✔
31	from hdx.utilities.path import (	1✔
32	progress_storing_folder,
33	script_dir_plus_file,
34	wheretostart_tempdir_batch,
35	)
36	from hdx.utilities.retriever import Retrieve	1✔
37
38	setup_logging()	1✔
39	logger = logging.getLogger(__name__)	1✔
40
41	lookup = "hdx-scraper-wfp-foodprices"	1✔
42	updated_by_script = "HDX Scraper: WFP Food Prices"	1✔
43
44
45	def main(	1✔
46	save: bool = False,
47	countryiso3s: str = "",
48	use_saved: bool = False,
49	save_wfp_rates: bool = True,
50	err_to_hdx: bool = False,
51	) -> None:
52	"""Generate datasets and create them in HDX
53
54	Args:
55	save (bool): Save all downloaded data. Defaults to False.
56	countryiso3s (str): Whether to limit to specific countries. Defaults to not limiting ("").
57	use_saved (bool): Use saved data. Defaults to False.
58	save_wfp_rates (bool): Save WFP FX rates data. Defaults to True.
59	err_to_hdx (bool): Whether to write any errors to HDX metadata. Defaults to False.
60
61	Returns:
62	None
63	"""
64	logger.info(f"##### {lookup} version {__version__} ####")	×
65	if not User.check_current_user_organization_access(	×
66	"3ecac442-7fed-448d-8f78-b385ef6f84e7", "create_dataset"
67	):
68	raise PermissionError("API Token does not give access to WFP organisation!")	×
69	with HDXErrorHandler(write_to_hdx=err_to_hdx) as error_handler:	×
70	with Download(	×
71	fail_on_missing_file=False,
72	extra_params_yaml=join(expanduser("~"), ".extraparams.yaml"),
73	extra_params_lookup=lookup,
74	) as token_downloader:
75	with Download(	×
76	use_env=False, rate_limit={"calls": 1, "period": 0.1}
77	) as downloader:
78	with wheretostart_tempdir_batch(lookup) as info:	×
79	if countryiso3s:	×
80	countryiso3s = countryiso3s.split(",")	×
81	else:
82	countryiso3s = None	×
83	folder = info["folder"]	×
84	batch = info["batch"]	×
85	retriever = Retrieve(	×
86	downloader, folder, "saved_data", folder, save, use_saved
87	)
88	params = {	×
89	"dialect": "sqlite",
90	"database": f"/{folder}/foodprices.sqlite",
91	}
92	with Database(**params) as database:	×
93	configuration = Configuration.read()	×
94	now = get_now(retriever)	×
95	wfp_api = WFPAPI(token_downloader, retriever)	×
96	wfp_api.update_retry_params(attempts=5, wait=3600)	×
97	wfp = WFPMappings(configuration, wfp_api, retriever)	×
98	iso3_to_showcase_url = wfp.read_region_mapping()	×
99	iso3_to_source = wfp.read_source_overrides()	×
100	countries = wfp.get_countries(countryiso3s)	×
101	logger.info(	×
102	f"Number of country datasets to upload: {len(countries)}"
103	)
104	commodity_to_category, dbcommodities = (	×
105	wfp.build_commodity_category_mapping()
106	)
107	if save_wfp_rates:	×
108	wfp_rates_folder = folder	×
109	else:
110	wfp_rates_folder = None	×
111	currencies = setup_currency(	×
112	now, retriever, wfp_api, wfp_rates_folder
113	)
114	dataset_generator = DatasetGenerator(	×
115	configuration,
116	folder,
117	iso3_to_showcase_url,
118	iso3_to_source,
119	currencies,
120	)
121	dbupdater = DBUpdater(configuration, database)	×
122	dbupdater.update_commodities(dbcommodities)	×
123
124	def process_country(country: Dict[str, str]) -> None:	×
125	countryiso3 = country["iso3"]	×
126	dataset, showcase = (	×
127	dataset_generator.get_dataset_and_showcase(countryiso3)
128	)
129	if not dataset:	×
130	return	×
131	wfp_food = WFPFood(	×
132	countryiso3,
133	configuration,
134	iso3_to_showcase_url.get(countryiso3),
135	iso3_to_source.get(countryiso3),
136	commodity_to_category,
137	)
138	dbmarkets = wfp_food.get_price_markets(wfp_api)	×
139	if not dbmarkets:	×
140	return	×
141	rows, markets, sources = wfp_food.generate_rows(dbmarkets)	×
142	dataset, qc_indicators = dataset_generator.complete_dataset(	×
143	countryiso3, dataset, rows, markets, sources
144	)
145	time_period = dataset.get_time_period()	×
146	hdx_url = dataset.get_hdx_url()	×
147	dbupdater.update_tables(	×
148	countryiso3,
149	time_period,
150	hdx_url,
151	dbmarkets,
152	)
153
154	snippet = f"Food Prices data for {country['name']}"	×
155	if dataset:	×
156	dataset.update_from_yaml(	×
157	script_dir_plus_file(
158	join("config", "hdx_dataset_static.yaml"),
159	main,
160	)
161	)
162	dataset["notes"] = dataset["notes"] % (snippet, "")	×
163	dataset.generate_quickcharts(	×
164	-1, indicators=qc_indicators
165	)
166	dataset.create_in_hdx(	×
167	remove_additional_resources=True,
168	hxl_update=False,
169	updated_by_script=updated_by_script,
170	batch=batch,
171	)
172	if showcase:	×
173	showcase.create_in_hdx()	×
174	showcase.add_dataset(dataset)	×
175	else:
176	logger.info(	×
177	f"{country['name']} does not have a showcase!"
178	)
179
180	for _, country in progress_storing_folder(	×
181	info, countries, "iso3"
182	):
183	process_country(country)	×
184
NEW 185	del hdx.location.int_timestamp._cache_timestamp_lookup	×
NEW 186	del Currency._cached_current_rates	×
NEW 187	del Currency._cached_historic_rates	×
NEW 188	del Currency._secondary_rates	×
NEW 189	del Currency._secondary_historic_rates	×
NEW 190	gc.collect()	×
191
UNCOV 192	table_data, start_date, end_date = (	×
193	dbupdater.get_data_from_tables()
194	)
195	global_prices_info = get_global_prices_rows(downloader, folder)	×
196	dataset, showcase = (	×
197	dataset_generator.generate_global_dataset_and_showcase(
198	global_prices_info, table_data, start_date, end_date
199	)
200	)
201	snippet = "Countries, Commodities and Markets data"	×
202	snippet2 = "The volume of data means that the actual Food Prices data is in country level datasets. "	×
203	dataset.update_from_yaml(	×
204	script_dir_plus_file(
205	join("config", "hdx_dataset_static.yaml"), main
206	)
207	)
208	dataset["notes"] = dataset["notes"] % (snippet, snippet2)	×
209	dataset.create_in_hdx(	×
210	remove_additional_resources=True,
211	hxl_update=False,
212	updated_by_script=updated_by_script,
213	batch=batch,
214	)
215	showcase.create_in_hdx()	×
216	showcase.add_dataset(dataset)	×
217
218	prices_resource_id = None	×
219	markets_resource_id = None	×
220	for resource in dataset.get_resources():	×
221	resource_name = resource["name"]	×
222	if resource_name == dataset_generator.global_name:	×
223	prices_resource_id = resource["id"]	×
224	elif resource_name == dataset_generator.global_markets_name:	×
225	markets_resource_id = resource["id"]	×
226	if prices_resource_id and markets_resource_id:	×
227	dataset_id = dataset["id"]	×
228	hapi_output = HAPIOutput(	×
229	configuration,
230	error_handler,
231	)
232	hapi_output.setup_admins(retriever, countryiso3s)	×
233	global_markets_info = table_data["DBMarket"]	×
234	hapi_output.process_markets(	×
235	global_markets_info, dataset_id, markets_resource_id
236	)
237	hapi_output.process_prices(	×
238	global_prices_info, dataset_id, prices_resource_id
239	)
240	hapi_dataset_generator = HAPIDatasetGenerator(	×
241	configuration,
242	global_markets_info,
243	global_prices_info,
244	)
245	dataset = hapi_dataset_generator.generate_prices_dataset(	×
246	folder,
247	)
248	if dataset:	×
249	dataset.update_from_yaml(	×
250	script_dir_plus_file(
251	join(
252	"config",
253	"hdx_hapi_dataset_static.yaml",
254	),
255	main,
256	)
257	)
258	dataset.create_in_hdx(	×
259	remove_additional_resources=False,
260	hxl_update=False,
261	updated_by_script=updated_by_script,
262	batch=batch,
263	)
NEW 264	logger.info("WFP global HAPI dataset created")	×
265
266
267	if __name__ == "__main__":	1✔
268	facade(	×
269	main,
270	user_agent_config_yaml=join(expanduser("~"), ".useragents.yaml"),
271	user_agent_lookup=lookup,
272	project_config_yaml=script_dir_plus_file(
273	join("config", "project_configuration.yaml"), main
274	),
275	)

OCHA-DAP / hdx-scraper-wfp-foodprices / 14117075234

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous