14844429278

Committed 05 May 2025 07:21PM UTC coverage: 78.703% (+1.3%) from 77.382%

Build # 14844429278

Build Type

push

github

Committed by

mcarans

Commit Message

Iterate over files repeatedly so as not to run out of memory

Run Details

144 of 158 new or added lines in 6 files covered. (91.14%)

2 existing lines in 1 file now uncovered.

728 of 925 relevant lines covered (78.7%)

0.79 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

91.67

/src/hdx/scraper/wfp/foodprices/world/global_markets.py

import logging
from glob import iglob
from typing import List, Optional

from hdx.utilities.downloader import Download

logger = logging.getLogger(__name__)


def get_markets(downloader: Download, folder: str) -> Optional[List]:
    filepaths = []
    for filepath in iglob(f"{folder}/wfp_markets*.csv", recursive=False):
        if any(x in filepath for x in ("_global",)):
            continue
        filepaths.append(filepath)

    rows = []
    for filepath in sorted(filepaths):
        countryiso3 = filepath[-7:-4].upper()
        _, iterator = downloader.get_tabular_rows(
            filepath, has_hxl=True, dict_form=True, encoding="utf-8"
        )
        logger.info(f"Reading markets from {countryiso3}: {filepath}")

        for row in iterator:
            market_id = row["market_id"]
            if market_id[0] == "#":
                continue
            rows.append(row)
    if len(rows) == 0:
        return None
    return rows

1	import logging	1✔
2	from glob import iglob	1✔
3	from typing import List, Optional	1✔
4
5	from hdx.utilities.downloader import Download	1✔
6
7	logger = logging.getLogger(__name__)	1✔
8
9
10	def get_markets(downloader: Download, folder: str) -> Optional[List]:	1✔
11	filepaths = []	1✔
12	for filepath in iglob(f"{folder}/wfp_markets*.csv", recursive=False):	1✔
13	if any(x in filepath for x in ("_global",)):	1✔
NEW 14	continue	×
15	filepaths.append(filepath)	1✔
16
17	rows = []	1✔
18	for filepath in sorted(filepaths):	1✔
19	countryiso3 = filepath[-7:-4].upper()	1✔
20	_, iterator = downloader.get_tabular_rows(	1✔
21	filepath, has_hxl=True, dict_form=True, encoding="utf-8"
22	)
23	logger.info(f"Reading markets from {countryiso3}: {filepath}")	1✔
24
25	for row in iterator:	1✔
26	market_id = row["market_id"]	1✔
27	if market_id[0] == "#":	1✔
28	continue	1✔
29	rows.append(row)	1✔
30	if len(rows) == 0:	1✔
NEW 31	return None	×
32	return rows	1✔

OCHA-DAP / hdx-scraper-wfp-foodprices / 14844429278

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous