4195006582

Build # 4195006582

Build Type

github

Committed by GitHub

Commit Message

Merge 14ceac9a6 into dc166625f

Pull Request Pull Request #918: ObjectStore - Reduce code duplication & more clone prevention

Run Details

177 of 177 new or added lines in 5 files covered. (100.0%)

24368 of 37519 relevant lines covered (64.95%)

43913.78 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/dozer-ingestion/src/connectors/object_store/helper.rs

use crate::errors::{ObjectStoreConnectorError, ObjectStoreObjectError};
use datafusion::datasource::file_format::csv::CsvFormat;
use datafusion::datasource::file_format::parquet::ParquetFormat;
use datafusion::datasource::listing::ListingOptions;
use dozer_types::ingestion_types::Table;
use std::sync::Arc;

pub fn map_listing_options(data_fusion_table: &Table) -> ListingOptions {
    match data_fusion_table.file_type.as_str() {
        "parquet" => {
            let format = ParquetFormat::new();
            ListingOptions::new(Arc::new(format))
                .with_file_extension(data_fusion_table.extension.clone())
        }
        "csv" => {
            let format = CsvFormat::default();
            ListingOptions::new(Arc::new(format))
                .with_file_extension(data_fusion_table.extension.clone())
        }
        _ => {
            let format = ParquetFormat::new();
            ListingOptions::new(Arc::new(format))
                .with_file_extension(data_fusion_table.extension.clone())
        }
    }
}

pub fn get_details<T>(details: &Option<T>) -> Result<&T, ObjectStoreConnectorError> {
    details
        .as_ref()
        .ok_or(ObjectStoreConnectorError::DataFusionStorageObjectError(
            ObjectStoreObjectError::MissingStorageDetails,
        ))
}

pub fn get_table<'a>(
    tables: &'a [Table],
    table_name: &str,
) -> Result<&'a Table, ObjectStoreConnectorError> {
    tables.iter().find(|table| table.name == table_name).ok_or(
        ObjectStoreConnectorError::DataFusionStorageObjectError(
            ObjectStoreObjectError::TableDefinitionNotFound,
        ),
    )
}

1	use crate::errors::{ObjectStoreConnectorError, ObjectStoreObjectError};
2	use datafusion::datasource::file_format::csv::CsvFormat;
3	use datafusion::datasource::file_format::parquet::ParquetFormat;
4	use datafusion::datasource::listing::ListingOptions;
5	use dozer_types::ingestion_types::Table;
6	use std::sync::Arc;
7		×
8	pub fn map_listing_options(data_fusion_table: &Table) -> ListingOptions {	×
9	match data_fusion_table.file_type.as_str() {	×
10	"parquet" => {	×
11	let format = ParquetFormat::new();	×
12	ListingOptions::new(Arc::new(format))	×
13	.with_file_extension(data_fusion_table.extension.clone())	×
14	}	×
15	"csv" => {	×
16	let format = CsvFormat::default();	×
17	ListingOptions::new(Arc::new(format))	×
18	.with_file_extension(data_fusion_table.extension.clone())	×
19	}
20	_ => {	×
21	let format = ParquetFormat::new();	×
22	ListingOptions::new(Arc::new(format))	×
23	.with_file_extension(data_fusion_table.extension.clone())	×
24	}
25	}	×
26	}	×
27
28	pub fn get_details<T>(details: &Option<T>) -> Result<&T, ObjectStoreConnectorError> {	×
29	details	×
30	.as_ref()	×
31	.ok_or(ObjectStoreConnectorError::DataFusionStorageObjectError(	×
32	ObjectStoreObjectError::MissingStorageDetails,	×
33	))	×
34	}	×
35
36	pub fn get_table<'a>(	×
37	tables: &'a [Table],	×
38	table_name: &str,	×
39	) -> Result<&'a Table, ObjectStoreConnectorError> {	×
40	tables.iter().find(\|table\| table.name == table_name).ok_or(	×
41	ObjectStoreConnectorError::DataFusionStorageObjectError(	×
42	ObjectStoreObjectError::TableDefinitionNotFound,	×
43	),	×
44	)	×
45	}	×

getdozer / dozer / 4195006582

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous