jay-dee7 commented on code in PR #25640:
URL: https://github.com/apache/datafusion/pull/25640#discussion_r4181442444
##########
datafusion-cli/src/functions.rs:
##########
@@ -463,6 +467,184 @@ impl TableFunctionImpl for ParquetMetadataFunc {
}
}
+/// PARQUET_FILE_METADATA table function
+#[derive(Debug)]
+pub struct ParquetFileMetadataFunc {}
+
+impl TableFunctionImpl for ParquetFileMetadataFunc {
+ fn call_with_args(&self, args: TableFunctionArgs) -> Result<Arc<dyn
TableProvider>> {
+ let filename = parquet_file_path("parquet_file_metadata",
args.exprs())?;
+
+ let mut reader = ParquetMetaDataReader::new();
+ reader.try_parse(&File::open(filename)?)?;
+ let footer_length = reader.metadata_size().map(|size| size as i64);
+ let metadata = reader.finish()?;
+ let file_metadata = metadata.file_metadata();
+
+ let key_values = file_metadata.key_value_metadata();
+ let mut key_value_metadata =
+ MapBuilder::new(None, StringBuilder::new(), StringBuilder::new());
+ for KeyValue { key, value } in key_values.into_iter().flatten() {
+ key_value_metadata.keys().append_value(key);
+ key_value_metadata.values().append_option(value.as_ref());
+ }
+ key_value_metadata.append(key_values.is_some())?;
+ let key_value_metadata = key_value_metadata.finish();
+
+ let schema = Arc::new(Schema::new(vec![
+ Field::new("filename", DataType::Utf8, true),
+ Field::new("created_by", DataType::Utf8, true),
+ Field::new("version", DataType::Int64, true),
+ Field::new("num_rows", DataType::Int64, true),
+ Field::new("num_row_groups", DataType::Int64, true),
+ Field::new(
+ "key_value_metadata",
+ key_value_metadata.data_type().clone(),
+ true,
+ ),
+ Field::new("footer_length", DataType::Int64, true),
+ ]));
+
+ let batch = RecordBatch::try_new(
+ Arc::clone(&schema),
+ vec![
+ Arc::new(StringArray::from(vec![filename])),
+ Arc::new(StringArray::from(vec![file_metadata.created_by()])),
+
Arc::new(Int64Array::from(vec![i64::from(file_metadata.version())])),
+ Arc::new(Int64Array::from(vec![file_metadata.num_rows()])),
+ Arc::new(Int64Array::from(vec![metadata.num_row_groups() as
i64])),
+ Arc::new(key_value_metadata),
+ Arc::new(Int64Array::from(vec![footer_length])),
+ ],
+ )?;
+
+ Ok(Arc::new(ParquetMetadataTable { schema, batch }))
+ }
+}
+
+/// Formats the min and max of page `idx` in `index` the same way as
+/// [`convert_parquet_statistics`]
+fn convert_page_min_max(
+ index: &ColumnIndexMetaData,
+ idx: usize,
+ converted_type: ConvertedType,
+) -> (Option<String>, Option<String>) {
+ fn primitive<T: ToString>(
+ index: &PrimitiveColumnIndex<T>,
+ idx: usize,
+ ) -> (Option<String>, Option<String>) {
+ (
+ index.min_value(idx).map(T::to_string),
+ index.max_value(idx).map(T::to_string),
+ )
+ }
+ let bytes = |val: &[u8]| match std::str::from_utf8(val) {
+ Ok(s) if converted_type == ConvertedType::UTF8 => s.to_string(),
+ _ => format!("{val:?}"),
+ };
+
+ match index {
Review Comment:
sure, will do, and thanks for being pro-active :)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
