ZENOTME commented on code in PR #837:
URL: https://github.com/apache/iceberg-rust/pull/837#discussion_r1896851693
##########
crates/iceberg/src/writer/file_writer/parquet_writer.rs:
##########
@@ -1169,4 +1172,187 @@ mod tests {
Ok(())
}
+
+ #[tokio::test]
+ async fn test_decimal_bound() -> Result<()> {
+ let temp_dir = TempDir::new().unwrap();
+ let file_io = FileIOBuilder::new_fs_io().build().unwrap();
+ let loccation_gen =
+
MockLocationGenerator::new(temp_dir.path().to_str().unwrap().to_string());
+ let file_name_gen =
+ DefaultFileNameGenerator::new("test".to_string(), None,
DataFileFormat::Parquet);
+
+ // test 1.1 and 2.2
+ let schema = Arc::new(
+ Schema::builder()
+ .with_fields(vec![NestedField::optional(
+ 0,
+ "decimal",
+ Type::Primitive(PrimitiveType::Decimal {
+ precision: 28,
+ scale: 10,
+ }),
+ )
+ .into()])
+ .build()
+ .unwrap(),
+ );
+ let arrow_schema: ArrowSchemaRef =
Arc::new(schema_to_arrow_schema(&schema).unwrap());
+ let mut pw = ParquetWriterBuilder::new(
+ WriterProperties::builder().build(),
+ schema.clone(),
+ file_io.clone(),
+ loccation_gen.clone(),
+ file_name_gen.clone(),
+ )
+ .build()
+ .await?;
+ let col0 = Arc::new(
+ Decimal128Array::from(vec![Some(22000000000), Some(11000000000)])
+ .with_data_type(DataType::Decimal128(28, 10)),
+ ) as ArrayRef;
+ let to_write = RecordBatch::try_new(arrow_schema.clone(),
vec![col0]).unwrap();
+ pw.write(&to_write).await?;
+ let res = pw.close().await?;
+ assert_eq!(res.len(), 1);
+ let data_file = res
+ .into_iter()
+ .next()
+ .unwrap()
+ .content(crate::spec::DataContentType::Data)
+ .partition(Struct::empty())
+ .build()
+ .unwrap();
+ assert_eq!(
+ data_file.upper_bounds().get(&0),
+ Some(Datum::decimal_with_precision(Decimal::new(22000000000_i64,
10), 28).unwrap())
+ .as_ref()
+ );
+ assert_eq!(
+ data_file.lower_bounds().get(&0),
+ Some(Datum::decimal_with_precision(Decimal::new(11000000000_i64,
10), 28).unwrap())
+ .as_ref()
+ );
+
+ // test -1.1 and -2.2
+ let schema = Arc::new(
+ Schema::builder()
+ .with_fields(vec![NestedField::optional(
+ 0,
+ "decimal",
+ Type::Primitive(PrimitiveType::Decimal {
+ precision: 28,
+ scale: 10,
+ }),
+ )
+ .into()])
+ .build()
+ .unwrap(),
+ );
+ let arrow_schema: ArrowSchemaRef =
Arc::new(schema_to_arrow_schema(&schema).unwrap());
+ let mut pw = ParquetWriterBuilder::new(
+ WriterProperties::builder().build(),
+ schema.clone(),
+ file_io.clone(),
+ loccation_gen.clone(),
+ file_name_gen.clone(),
+ )
+ .build()
+ .await?;
+ let col0 = Arc::new(
+ Decimal128Array::from(vec![Some(-22000000000), Some(-11000000000)])
+ .with_data_type(DataType::Decimal128(28, 10)),
+ ) as ArrayRef;
+ let to_write = RecordBatch::try_new(arrow_schema.clone(),
vec![col0]).unwrap();
+ pw.write(&to_write).await?;
+ let res = pw.close().await?;
+ assert_eq!(res.len(), 1);
+ let data_file = res
+ .into_iter()
+ .next()
+ .unwrap()
+ .content(crate::spec::DataContentType::Data)
+ .partition(Struct::empty())
+ .build()
+ .unwrap();
+ assert_eq!(
+ data_file.upper_bounds().get(&0),
+ Some(Datum::decimal_with_precision(Decimal::new(-11000000000_i64,
10), 28).unwrap())
+ .as_ref()
+ );
+ assert_eq!(
+ data_file.lower_bounds().get(&0),
+ Some(Datum::decimal_with_precision(Decimal::new(-22000000000_i64,
10), 28).unwrap())
+ .as_ref()
+ );
+
+ // test max and min for scale 38
+ let schema = Arc::new(
+ Schema::builder()
+ .with_fields(vec![NestedField::optional(
+ 0,
+ "decimal",
+ Type::Primitive(PrimitiveType::Decimal {
+ precision: 38,
+ scale: 0,
+ }),
+ )
+ .into()])
+ .build()
+ .unwrap(),
+ );
+ let arrow_schema: ArrowSchemaRef =
Arc::new(schema_to_arrow_schema(&schema).unwrap());
+ let mut pw = ParquetWriterBuilder::new(
+ WriterProperties::builder().build(),
+ schema,
+ file_io.clone(),
+ loccation_gen,
+ file_name_gen,
+ )
+ .build()
+ .await?;
+ let col0 = Arc::new(
+ Decimal128Array::from(vec![
+ Some(99999999999999999999999999999999999999_i128),
+ Some(-99999999999999999999999999999999999999_i128),
+ ])
+ .with_data_type(DataType::Decimal128(38, 0)),
+ ) as ArrayRef;
+ let to_write = RecordBatch::try_new(arrow_schema.clone(),
vec![col0]).unwrap();
+ pw.write(&to_write).await?;
+ let res = pw.close().await?;
+ assert_eq!(res.len(), 1);
+ let data_file = res
+ .into_iter()
+ .next()
+ .unwrap()
+ .content(crate::spec::DataContentType::Data)
+ .partition(Struct::empty())
+ .build()
+ .unwrap();
+ assert_eq!(
+ data_file.upper_bounds().get(&0),
+ Some(Datum::new(
+ PrimitiveType::Decimal {
+ precision: 38,
+ scale: 0
+ },
+
PrimitiveLiteral::Int128(99999999999999999999999999999999999999_i128)
Review Comment:
But from the definition of iceberg spec, the precision of decimal can be 38.
So I think for here we can't prevent user write decimal with precision larger
than 28 but smaller than 38. https://iceberg.apache.org/spec/#primitive-types
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscr...@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscr...@iceberg.apache.org
For additional commands, e-mail: issues-h...@iceberg.apache.org
