mbutrovich commented on code in PR #3632:
URL: https://github.com/apache/iceberg-rust/pull/3632#discussion_r4588298317
##########
crates/iceberg/src/arrow/int96.rs:
##########
@@ -348,39 +359,174 @@ mod tests {
let arrow_schema = Arc::new(ArrowSchema::new(vec![
Field::new(
"ts",
- DataType::Timestamp(TimeUnit::Nanosecond, Some("UTC".into())),
+ DataType::Timestamp(TimeUnit::Nanosecond,
Some(UTC_TIME_ZONE.into())),
true,
)
.with_metadata(field_id_meta(1)),
]));
- assert!(coerce_int96_timestamps(&arrow_schema, &iceberg).is_none());
+ assert!(
+ coerce_int96_timestamps(&arrow_schema, &iceberg, &parquet)
+ .unwrap()
+ .is_none()
+ );
}
#[test]
fn test_no_coercion_when_already_microsecond() {
+ let parquet = parquet_schema("message schema { optional int96 ts;
required int32 id; }");
let arrow_schema = Arc::new(ArrowSchema::new(vec![
Field::new("ts", DataType::Timestamp(TimeUnit::Microsecond, None),
true)
.with_metadata(field_id_meta(1)),
Field::new("id", DataType::Int32,
false).with_metadata(field_id_meta(2)),
]));
let iceberg = iceberg_schema_with_timestamp();
- assert!(coerce_int96_timestamps(&arrow_schema, &iceberg).is_none());
+ assert!(
+ coerce_int96_timestamps(&arrow_schema, &iceberg, &parquet)
+ .unwrap()
+ .is_none()
+ );
+ }
+
+ #[test]
+ fn test_no_coercion_without_int96() {
+ let parquet =
+ parquet_schema("message schema { optional int64 ts
(TIMESTAMP(NANOS,false)) = 1; }");
+ let arrow_schema =
+ Arc::new(parquet::arrow::parquet_to_arrow_schema(&parquet,
None).unwrap());
+ let iceberg = iceberg_schema_with_timestamp();
+
+ assert_eq!(
+ coerce_int96_timestamps(&arrow_schema, &iceberg,
&parquet).unwrap(),
+ None
+ );
+ }
+
+ #[test]
+ fn test_coercion_rejects_list_view() {
+ let parquet = parquet_schema(
+ "message schema {
+ optional group items (LIST) {
+ repeated group list { optional int32 element; }
+ }
+ optional int96 ts = 1;
+ }",
+ );
+ let iceberg = Schema::builder()
+ .with_fields(vec![
+ NestedField::optional(1, "ts",
Type::Primitive(PrimitiveType::Timestamptz)).into(),
+ ])
+ .build()
+ .unwrap();
+ let list_type = DataType::ListView(Arc::new(Field::new("element",
DataType::Int32, true)));
+ let arrow_schema = Arc::new(ArrowSchema::new(vec![
+ Field::new("items", list_type.clone(), true),
+ Field::new("ts", DataType::Timestamp(TimeUnit::Nanosecond, None),
true)
+ .with_metadata(field_id_meta(1)),
+ ]));
+
+ let error = coerce_int96_timestamps(&arrow_schema, &iceberg,
&parquet).unwrap_err();
+ assert_eq!(
+ (error.kind(), error.message()),
+ (
+ crate::ErrorKind::DataInvalid,
+ format!("Cannot visit Arrow data type: {list_type}").as_str(),
+ )
+ );
+ }
+
+ #[test]
+ fn test_coercion_rejects_mismatched_leaf_counts() {
+ let iceberg = iceberg_schema_with_timestamp();
+ for (message, expected_message) in [
+ (
+ "message schema { optional int96 ts; required int32 id; }",
+ "Arrow schema has fewer primitive leaves than the Parquet
schema has columns",
+ ),
+ (
+ "message schema { optional int96 ts; }",
+ "Missing Parquet column for Arrow primitive",
+ ),
+ ] {
+ let parquet = parquet_schema(message);
+ let fields = [
+ Field::new("ts", DataType::Timestamp(TimeUnit::Nanosecond,
None), true)
+ .with_metadata(field_id_meta(1)),
+ Field::new("id", DataType::Int32,
false).with_metadata(field_id_meta(2)),
+ ];
+ let leaf_count = if parquet.num_columns() == 2 { 1 } else { 2 };
+ let arrow_schema =
Arc::new(ArrowSchema::new(fields[..leaf_count].to_vec()));
Review Comment:
The number of Arrow fields in each case comes from `if parquet.num_columns()
== 2 { 1 } else { 2 }`. A reader has to work that out to see which case is
which. Could the table list the Arrow leaf count next to the Parquet message
instead?
```suggestion
let fields = [
Field::new("ts", DataType::Timestamp(TimeUnit::Nanosecond,
None), true)
.with_metadata(field_id_meta(1)),
Field::new("id", DataType::Int32,
false).with_metadata(field_id_meta(2)),
];
for (message, arrow_leaves, expected_message) in [
(
"message schema { optional int96 ts; required int32 id; }",
1,
"Arrow schema has fewer primitive leaves than the Parquet
schema has columns",
),
(
"message schema { optional int96 ts; }",
2,
"Missing Parquet column for Arrow primitive",
),
] {
let parquet = parquet_schema(message);
let arrow_schema =
Arc::new(ArrowSchema::new(fields[..arrow_leaves].to_vec()));
```
##########
crates/iceberg/src/arrow/int96.rs:
##########
@@ -34,6 +36,7 @@ use crate::{Error, ErrorKind};
/// arrow-rs defaults INT96 to `Timestamp(Nanosecond)`, which overflows i64
for dates outside
/// ~1677-2262. We use arrow-rs's schema hint mechanism to read INT96 at the
resolution
/// specified by the Iceberg schema (`timestamp` → microsecond, `timestamp_ns`
→ nanosecond).
+/// The Arrow schema must contain all physical columns in file order, before
projection.
Review Comment:
This doc comment still covers only the resolution. The timezone is the
actual fix for #3611, and why it's needed isn't obvious from the code.
Projection converts the file's Arrow type to an Iceberg type and [drops the
leaf](https://github.com/apache/iceberg-rust/blob/eeb1e41f959589347ac98a0575a325cdc923f6e9/crates/iceberg/src/arrow/reader/projection.rs#L239-L246)
when that type doesn't match the table's. `Timestamp(Microsecond, None)`
[converts to
`timestamp`](https://github.com/apache/iceberg-rust/blob/eeb1e41f959589347ac98a0575a325cdc923f6e9/crates/iceberg/src/arrow/schema.rs#L712-L723),
not `timestamptz`. Could we say that here?
```suggestion
/// specified by the Iceberg schema (`timestamp` → microsecond,
`timestamp_ns` → nanosecond).
/// For `timestamptz` and `timestamptz_ns`, the hint also sets the UTC
timezone. Without it,
/// projection sees a `timestamp` column where the table expects
`timestamptz`, drops the
/// column, and the read returns NULLs.
/// The Arrow schema must contain all physical columns in file order, before
projection.
```
##########
crates/iceberg/src/arrow/reader/pipeline.rs:
##########
@@ -2957,46 +2956,330 @@ mod tests {
#[tokio::test]
async fn test_read_int96_timestamps_with_field_ids() {
+ for timestamp_type in [PrimitiveType::Timestamp,
PrimitiveType::Timestamptz] {
+ assert_int96_microsecond_projections(timestamp_type, true,
None).await;
+ }
+ }
+
+ #[tokio::test]
+ async fn test_read_int96_timestamps_without_field_ids() {
+ for timestamp_type in [PrimitiveType::Timestamp,
PrimitiveType::Timestamptz] {
+ assert_int96_microsecond_projections(timestamp_type, false,
None).await;
+ }
+ }
+
+ #[tokio::test]
+ async fn test_read_int96_timestamps_with_name_mapping() {
+ let name_mapping = Arc::new(NameMapping::new(vec![
+ MappedField::new(Some(1), vec!["ts".to_string()], vec![]),
+ MappedField::new(Some(2), vec!["id".to_string()], vec![]),
+ ]));
+ for timestamp_type in [PrimitiveType::Timestamp,
PrimitiveType::Timestamptz] {
+ assert_int96_microsecond_projections(timestamp_type, false,
Some(name_mapping.clone()))
+ .await;
+ }
+ }
+
+ async fn assert_int96_microsecond_projections(
+ timestamp_type: PrimitiveType,
+ with_field_ids: bool,
+ name_mapping: Option<Arc<NameMapping>>,
+ ) {
+ use arrow_array::TimestampMicrosecondArray;
+
+ use crate::arrow::schema::UTC_TIME_ZONE;
+ use crate::arrow::schema_to_arrow_schema;
+
+ let timezone = (timestamp_type ==
PrimitiveType::Timestamptz).then_some(UTC_TIME_ZONE);
let schema = Arc::new(
Schema::builder()
.with_schema_id(1)
.with_fields(vec![
- NestedField::optional(1, "ts",
Type::Primitive(PrimitiveType::Timestamp))
- .into(),
+ NestedField::optional(1, "ts",
Type::Primitive(timestamp_type)).into(),
NestedField::required(2, "id",
Type::Primitive(PrimitiveType::Int)).into(),
])
.build()
.unwrap(),
);
+ let tmp_dir = TempDir::new().unwrap();
+ let (file_path, expected_micros) = write_int96_parquet_file(
+ tmp_dir.path().to_str().unwrap(),
+ "timestamps.parquet",
+ with_field_ids,
+ );
+ let ids: Vec<i32> = (0..expected_micros.len() as i32).collect();
+ let expected_batch =
+
RecordBatch::try_new(Arc::new(schema_to_arrow_schema(&schema).unwrap()), vec![
+ Arc::new(
+
TimestampMicrosecondArray::from(expected_micros).with_timezone_opt(timezone),
+ ),
+ Arc::new(Int32Array::from(ids)),
+ ])
+ .unwrap();
+ for (field_ids, column_indices) in [
+ (vec![1], vec![0]),
+ (vec![1, 2], vec![0, 1]),
+ (vec![2, 1], vec![1, 0]),
+ ] {
+ let actual = read_int96_batches_with_name_mapping(
+ &file_path,
+ schema.clone(),
+ field_ids,
+ name_mapping.clone(),
+ )
+ .await;
+ let expected =
vec![expected_batch.project(&column_indices).unwrap()];
+ assert_eq!(actual, expected);
+ }
+ }
+ #[tokio::test]
+ async fn test_read_int96_nanosecond_timestamps() {
+ for timestamp_type in [PrimitiveType::TimestampNs,
PrimitiveType::TimestamptzNs] {
+ for with_field_ids in [true, false] {
+ assert_int96_nanosecond_projections(timestamp_type.clone(),
with_field_ids).await;
+ }
+ }
+ }
+
+ async fn assert_int96_nanosecond_projections(
+ timestamp_type: PrimitiveType,
+ with_field_ids: bool,
+ ) {
+ use arrow_array::TimestampNanosecondArray;
+ use parquet::data_type::{Int32Type, Int96, Int96Type};
+ use parquet::file::writer::SerializedFileWriter;
+ use parquet::schema::parser::parse_message_type;
+
+ use crate::arrow::schema::UTC_TIME_ZONE;
+ use crate::arrow::schema_to_arrow_schema;
+
+ let timezone = (timestamp_type ==
PrimitiveType::TimestamptzNs).then_some(UTC_TIME_ZONE);
+ let schema = Arc::new(
+ Schema::builder()
+ .with_fields(vec![
+ NestedField::optional(1, "ts",
Type::Primitive(timestamp_type)).into(),
+ NestedField::required(2, "id",
Type::Primitive(PrimitiveType::Int)).into(),
+ ])
+ .build()
+ .unwrap(),
+ );
+ let message = if with_field_ids {
+ "message schema { optional int96 ts = 1; required int32 id = 2; }"
+ } else {
+ "message schema { optional int96 ts; required int32 id; }"
+ };
+ let parquet_schema = parse_message_type(message).unwrap();
+ let name_mapping = (!with_field_ids).then(|| {
+ Arc::new(NameMapping::new(vec![
+ MappedField::new(Some(1), vec!["ts".to_string()], vec![]),
+ MappedField::new(Some(2), vec!["id".to_string()], vec![]),
+ ]))
+ });
+ let nanos = vec![Some(1), None, Some(2_000_006_001_i64)];
+ let int96_values: Vec<Int96> = nanos
+ .iter()
+ .flatten()
+ .map(|nanos| {
+ let mut value = Int96::new();
+ value.set_data(
+ *nanos as u32,
+ (nanos >> 32) as u32,
+ UNIX_EPOCH_JULIAN as u32,
+ );
+ value
+ })
+ .collect();
+ let ids = vec![0, 1, 2];
let tmp_dir = TempDir::new().unwrap();
- let table_location = tmp_dir.path().to_str().unwrap().to_string();
- let (file_path, expected_micros) =
- write_int96_parquet_file(&table_location, "with_ids.parquet",
true);
+ let file_path = tmp_dir.path().join("nanosecond_timestamps.parquet");
+ let mut writer = SerializedFileWriter::new(
+ File::create(&file_path).unwrap(),
+ Arc::new(parquet_schema),
+ Default::default(),
+ )
+ .unwrap();
+ let mut row_group = writer.next_row_group().unwrap();
+ let mut column = row_group.next_column().unwrap().unwrap();
+ column
+ .typed::<Int96Type>()
+ .write_batch(&int96_values, Some(&[1, 0, 1]), None)
+ .unwrap();
+ column.close().unwrap();
+ let mut column = row_group.next_column().unwrap().unwrap();
+ column
+ .typed::<Int32Type>()
+ .write_batch(&ids, None, None)
+ .unwrap();
+ column.close().unwrap();
+ row_group.close().unwrap();
+ writer.close().unwrap();
- assert_int96_read_matches(&file_path, schema, vec![1, 2],
&expected_micros).await;
+ let expected_batch =
+
RecordBatch::try_new(Arc::new(schema_to_arrow_schema(&schema).unwrap()), vec![
+
Arc::new(TimestampNanosecondArray::from(nanos).with_timezone_opt(timezone)),
+ Arc::new(Int32Array::from(ids)),
+ ])
+ .unwrap();
+ for (field_ids, column_indices) in [
+ (vec![1], vec![0]),
+ (vec![1, 2], vec![0, 1]),
+ (vec![2, 1], vec![1, 0]),
+ ] {
+ let actual = read_int96_batches_with_name_mapping(
+ file_path.to_str().unwrap(),
+ schema.clone(),
+ field_ids,
+ name_mapping.clone(),
+ )
+ .await;
+ let expected =
vec![expected_batch.project(&column_indices).unwrap()];
+ assert_eq!(actual, expected);
+ }
}
#[tokio::test]
- async fn test_read_int96_timestamps_without_field_ids() {
+ async fn test_read_int96_with_mixed_timestamp_types_in_struct() {
Review Comment:
Every nested INT96 test in this PR embeds field IDs, including this one and
the list, map, and struct tests. Spark-migrated files have INT96 inside structs
and no field IDs, so the reader resolves them through a name mapping. That's
the layout Comet reads.
Could this test take `with_field_ids` like
[`assert_int96_nanosecond_projections`](https://github.com/apache/iceberg-rust/blob/47800f10250915dd141757bf906cf882176208a7/crates/iceberg/src/arrow/reader/pipeline.rs#L3067-L3078)
does? Without IDs, it would drop the `= N` IDs from the message and pass a
nested name mapping (`data` with children `micros` and `legacy`) to
`read_int96_batches_with_name_mapping`. I tried that on this branch and the
expected batch matches for all three projections.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]