Xuanwo commented on code in PR #837:
URL: https://github.com/apache/iceberg-rust/pull/837#discussion_r1896852907
##########
crates/iceberg/src/writer/file_writer/parquet_writer.rs:
##########
@@ -1169,4 +1172,187 @@ mod tests {
Ok(())
}
+
+ #[tokio::test]
+ async fn test_decimal_bound() -> Result<()> {
+ let temp_dir = TempDir::new().unwrap();
+ let file_io = FileIOBuilder::new_fs_io().build().unwrap();
+ let loccation_gen =
+
MockLocationGenerator::new(temp_dir.path().to_str().unwrap().to_string());
+ let file_name_gen =
+ DefaultFileNameGenerator::new("test".to_string(), None,
DataFileFormat::Parquet);
+
+ // test 1.1 and 2.2
+ let schema = Arc::new(
+ Schema::builder()
+ .with_fields(vec![NestedField::optional(
+ 0,
+ "decimal",
+ Type::Primitive(PrimitiveType::Decimal {
+ precision: 28,
+ scale: 10,
+ }),
+ )
+ .into()])
+ .build()
+ .unwrap(),
+ );
+ let arrow_schema: ArrowSchemaRef =
Arc::new(schema_to_arrow_schema(&schema).unwrap());
+ let mut pw = ParquetWriterBuilder::new(
+ WriterProperties::builder().build(),
+ schema.clone(),
+ file_io.clone(),
+ loccation_gen.clone(),
+ file_name_gen.clone(),
+ )
+ .build()
+ .await?;
+ let col0 = Arc::new(
+ Decimal128Array::from(vec![Some(22000000000), Some(11000000000)])
+ .with_data_type(DataType::Decimal128(28, 10)),
+ ) as ArrayRef;
+ let to_write = RecordBatch::try_new(arrow_schema.clone(),
vec![col0]).unwrap();
+ pw.write(&to_write).await?;
+ let res = pw.close().await?;
+ assert_eq!(res.len(), 1);
+ let data_file = res
+ .into_iter()
+ .next()
+ .unwrap()
+ .content(crate::spec::DataContentType::Data)
+ .partition(Struct::empty())
+ .build()
+ .unwrap();
+ assert_eq!(
+ data_file.upper_bounds().get(&0),
+ Some(Datum::decimal_with_precision(Decimal::new(22000000000_i64,
10), 28).unwrap())
+ .as_ref()
+ );
+ assert_eq!(
+ data_file.lower_bounds().get(&0),
+ Some(Datum::decimal_with_precision(Decimal::new(11000000000_i64,
10), 28).unwrap())
+ .as_ref()
+ );
+
+ // test -1.1 and -2.2
+ let schema = Arc::new(
+ Schema::builder()
+ .with_fields(vec![NestedField::optional(
+ 0,
+ "decimal",
+ Type::Primitive(PrimitiveType::Decimal {
+ precision: 28,
+ scale: 10,
+ }),
+ )
+ .into()])
+ .build()
+ .unwrap(),
+ );
+ let arrow_schema: ArrowSchemaRef =
Arc::new(schema_to_arrow_schema(&schema).unwrap());
+ let mut pw = ParquetWriterBuilder::new(
+ WriterProperties::builder().build(),
+ schema.clone(),
+ file_io.clone(),
+ loccation_gen.clone(),
+ file_name_gen.clone(),
+ )
+ .build()
+ .await?;
+ let col0 = Arc::new(
+ Decimal128Array::from(vec![Some(-22000000000), Some(-11000000000)])
+ .with_data_type(DataType::Decimal128(28, 10)),
+ ) as ArrayRef;
+ let to_write = RecordBatch::try_new(arrow_schema.clone(),
vec![col0]).unwrap();
+ pw.write(&to_write).await?;
+ let res = pw.close().await?;
+ assert_eq!(res.len(), 1);
+ let data_file = res
+ .into_iter()
+ .next()
+ .unwrap()
+ .content(crate::spec::DataContentType::Data)
+ .partition(Struct::empty())
+ .build()
+ .unwrap();
+ assert_eq!(
+ data_file.upper_bounds().get(&0),
+ Some(Datum::decimal_with_precision(Decimal::new(-11000000000_i64,
10), 28).unwrap())
+ .as_ref()
+ );
+ assert_eq!(
+ data_file.lower_bounds().get(&0),
+ Some(Datum::decimal_with_precision(Decimal::new(-22000000000_i64,
10), 28).unwrap())
+ .as_ref()
+ );
+
+ // test max and min for scale 38
+ let schema = Arc::new(
+ Schema::builder()
+ .with_fields(vec![NestedField::optional(
+ 0,
+ "decimal",
+ Type::Primitive(PrimitiveType::Decimal {
+ precision: 38,
+ scale: 0,
+ }),
+ )
+ .into()])
+ .build()
+ .unwrap(),
+ );
+ let arrow_schema: ArrowSchemaRef =
Arc::new(schema_to_arrow_schema(&schema).unwrap());
+ let mut pw = ParquetWriterBuilder::new(
+ WriterProperties::builder().build(),
+ schema,
+ file_io.clone(),
+ loccation_gen,
+ file_name_gen,
+ )
+ .build()
+ .await?;
+ let col0 = Arc::new(
+ Decimal128Array::from(vec![
+ Some(99999999999999999999999999999999999999_i128),
+ Some(-99999999999999999999999999999999999999_i128),
+ ])
+ .with_data_type(DataType::Decimal128(38, 0)),
+ ) as ArrayRef;
+ let to_write = RecordBatch::try_new(arrow_schema.clone(),
vec![col0]).unwrap();
+ pw.write(&to_write).await?;
+ let res = pw.close().await?;
+ assert_eq!(res.len(), 1);
+ let data_file = res
+ .into_iter()
+ .next()
+ .unwrap()
+ .content(crate::spec::DataContentType::Data)
+ .partition(Struct::empty())
+ .build()
+ .unwrap();
+ assert_eq!(
+ data_file.upper_bounds().get(&0),
+ Some(Datum::new(
+ PrimitiveType::Decimal {
+ precision: 38,
+ scale: 0
+ },
+
PrimitiveLiteral::Int128(99999999999999999999999999999999999999_i128)
Review Comment:
Yep, that is exactly the issue I raised here
https://github.com/apache/iceberg-rust/issues/669
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscr...@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscr...@iceberg.apache.org
For additional commands, e-mail: issues-h...@iceberg.apache.org
