anoopj commented on code in PR #2966:
URL: https://github.com/apache/iceberg-rust/pull/2966#discussion_r3744919028
##########
crates/iceberg/src/arrow/reader/pipeline.rs:
##########
@@ -909,6 +959,257 @@ mod tests {
);
}
+ /// Writes a plain (unencrypted) single-column Int32 "id" parquet file
with the
+ /// given extra Arrow fields/columns appended, returning the file path.
+ fn write_plain_parquet(
+ dir: &str,
+ name: &str,
+ extra_fields: Vec<Field>,
+ extra_columns: Vec<ArrayRef>,
+ ) -> String {
+ let mut fields =
+ vec![
+ Field::new("id", DataType::Int32,
false).with_metadata(HashMap::from([(
+ PARQUET_FIELD_ID_META_KEY.to_string(),
+ "1".to_string(),
+ )])),
+ ];
+ fields.extend(extra_fields);
+ let arrow_schema = Arc::new(ArrowSchema::new(fields));
+
+ let mut columns: Vec<ArrayRef> =
vec![Arc::new(Int32Array::from(vec![1, 2, 3]))];
+ columns.extend(extra_columns);
+ let batch = RecordBatch::try_new(arrow_schema.clone(),
columns).unwrap();
+
+ let file_path = format!("{dir}/{name}");
+ let file = File::create(&file_path).unwrap();
+ let props = WriterProperties::builder()
+ .set_compression(Compression::SNAPPY)
+ .build();
+ let mut writer = ArrowWriter::try_new(file, arrow_schema,
Some(props)).unwrap();
+ writer.write(&batch).unwrap();
+ writer.close().unwrap();
+ file_path
+ }
+
+ fn last_updated_seq_task(
+ file_path: String,
+ first_row_id: Option<i64>,
+ data_sequence_number: Option<i64>,
+ ) -> FileScanTask {
+ use
crate::metadata_columns::RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER;
+
+ let schema = Arc::new(
+ Schema::builder()
+ .with_schema_id(1)
+ .with_fields(vec![
+ NestedField::required(1, "id",
Type::Primitive(PrimitiveType::Int)).into(),
+ ])
+ .build()
+ .unwrap(),
+ );
+
+ FileScanTask::builder()
+
.with_file_size_in_bytes(std::fs::metadata(&file_path).unwrap().len())
+ .with_start(0)
+ .with_length(0)
+ .with_data_file_path(file_path)
+ .with_data_file_format(DataFileFormat::Parquet)
+ .with_schema(schema)
+ .with_project_field_ids(vec![1,
RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER])
+ .with_first_row_id(first_row_id)
+ .with_data_sequence_number(data_sequence_number)
+ .with_case_sensitive(false)
+ .build()
+ }
+
+ #[tokio::test]
+ async fn test_last_updated_sequence_number_null_when_no_first_row_id() {
+ let tmp_dir = TempDir::new().unwrap();
+ let dir = tmp_dir.path().to_str().unwrap();
+ let file_path = write_plain_parquet(dir, "no_first_row_id.parquet",
vec![], vec![]);
+
+ // A file with a null first_row_id (v1/v2, or a pre-upgrade v3
snapshot) produces
+ // a null _last_updated_sequence_number column, even though it has a
data
+ // sequence number; the spec gates both lineage columns on
first_row_id.
Review Comment:
Youre correct. Reworded.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
