rdblue commented on code in PR #7886:
URL: https://github.com/apache/iceberg/pull/7886#discussion_r1264210910
##########
spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/TestSparkV2Filters.java:
##########
@@ -373,4 +386,371 @@ public void testNotIn() {
Expressions.and(Expressions.notNull("col"), Expressions.notIn("col",
1, 2));
Assert.assertEquals("Expressions should match", expected.toString(),
actual.toString());
}
+
+ @Test
+ public void testDateToYears() {
+ ScalarFunction<Integer> dateToYearsFunc = new
YearsFunction.DateToYearsFunction();
+ UserDefinedScalarFunc udf =
+ new UserDefinedScalarFunc(
+ dateToYearsFunc.name(),
+ dateToYearsFunc.canonicalName(),
+ expressions(FieldReference.apply("col1")));
+ testUDF(udf, Expressions.year("col1"), dateToYears("2023-06-25"),
DataTypes.IntegerType);
+ }
+
+ @Test
+ public void testTsToYears() {
+ ScalarFunction<Integer> tsToYearsFunc = new
YearsFunction.TimestampToYearsFunction();
+ UserDefinedScalarFunc udf =
+ new UserDefinedScalarFunc(
+ tsToYearsFunc.name(),
+ tsToYearsFunc.canonicalName(),
+ expressions(FieldReference.apply("col1")));
+ testUDF(
+ udf,
+ Expressions.year("col1"),
+ timestampToYears("2023-12-03T10:15:30+01:00"),
+ DataTypes.IntegerType);
+ }
+
+ @Test
+ public void testTsNtzToYears() {
+ ScalarFunction<Integer> tsNtzToYearsFunc = new
YearsFunction.TimestampNtzToYearsFunction();
+ UserDefinedScalarFunc udf =
+ new UserDefinedScalarFunc(
+ tsNtzToYearsFunc.name(),
+ tsNtzToYearsFunc.canonicalName(),
+ expressions(FieldReference.apply("col1")));
+ testUDF(
+ udf,
+ Expressions.year("col1"),
+ timestampNtzToYears("2023-06-25T13:15:30"),
+ DataTypes.IntegerType);
+ }
+
+ @Test
+ public void testDateToMonths() {
+ ScalarFunction<Integer> dateToMonthsFunc = new
MonthsFunction.DateToMonthsFunction();
+ UserDefinedScalarFunc udf =
+ new UserDefinedScalarFunc(
+ dateToMonthsFunc.name(),
+ dateToMonthsFunc.canonicalName(),
+ expressions(FieldReference.apply("col1")));
+ testUDF(udf, Expressions.month("col1"), dateToMonths("2023-06-25"),
DataTypes.IntegerType);
+ }
+
+ @Test
+ public void testTsToMonths() {
+ ScalarFunction<Integer> tsToMonthsFunc = new
MonthsFunction.TimestampToMonthsFunction();
+ UserDefinedScalarFunc udf =
+ new UserDefinedScalarFunc(
+ tsToMonthsFunc.name(),
+ tsToMonthsFunc.canonicalName(),
+ expressions(FieldReference.apply("col1")));
+ testUDF(
+ udf,
+ Expressions.month("col1"),
+ timestampToMonths("2023-12-03T10:15:30+01:00"),
+ DataTypes.IntegerType);
+ }
+
+ @Test
+ public void testTsNtzToMonths() {
+ ScalarFunction<Integer> tsNtzToMonthsFunc = new
MonthsFunction.TimestampNtzToMonthsFunction();
+ UserDefinedScalarFunc udf =
+ new UserDefinedScalarFunc(
+ tsNtzToMonthsFunc.name(),
+ tsNtzToMonthsFunc.canonicalName(),
+ expressions(FieldReference.apply("col1")));
+ testUDF(
+ udf,
+ Expressions.month("col1"),
+ timestampNtzToMonths("2023-12-03T10:15:30"),
+ DataTypes.IntegerType);
+ }
+
+ @Test
+ public void testDateToDays() {
+ ScalarFunction<Integer> dateToDayFunc = new
DaysFunction.DateToDaysFunction();
+ UserDefinedScalarFunc udf =
+ new UserDefinedScalarFunc(
+ dateToDayFunc.name(),
+ dateToDayFunc.canonicalName(),
+ expressions(FieldReference.apply("col1")));
+ testUDF(udf, Expressions.day("col1"), dateToDays("2023-06-25"),
DataTypes.IntegerType);
+ }
+
+ @Test
+ public void testTsToDays() {
+ ScalarFunction<Integer> tsToDaysFunc = new
DaysFunction.TimestampToDaysFunction();
+ UserDefinedScalarFunc udf =
+ new UserDefinedScalarFunc(
+ tsToDaysFunc.name(),
+ tsToDaysFunc.canonicalName(),
+ expressions(FieldReference.apply("col1")));
+ testUDF(
+ udf,
+ Expressions.day("col1"),
+ timestampToDays("2023-12-03T10:15:30+01:00"),
+ DataTypes.IntegerType);
+ }
+
+ @Test
+ public void testTsNtzToDays() {
+ ScalarFunction<Integer> tsNtzToDaysFunc = new
DaysFunction.TimestampNtzToDaysFunction();
+ UserDefinedScalarFunc udf =
+ new UserDefinedScalarFunc(
+ tsNtzToDaysFunc.name(),
+ tsNtzToDaysFunc.canonicalName(),
+ expressions(FieldReference.apply("col1")));
+ testUDF(
+ udf,
+ Expressions.day("col1"),
+ timestampNtzToDays("2023-12-03T10:15:30"),
+ DataTypes.IntegerType);
+ }
+
+ @Test
+ public void testTsToHours() {
+ ScalarFunction<Integer> tsToHourFunc = new
HoursFunction.TimestampToHoursFunction();
+ UserDefinedScalarFunc udf =
+ new UserDefinedScalarFunc(
+ tsToHourFunc.name(),
+ tsToHourFunc.canonicalName(),
+ expressions(FieldReference.apply("col1")));
+ testUDF(
+ udf,
+ Expressions.hour("col1"),
+ timestampToHours("2023-12-03T10:15:30+01:00"),
+ DataTypes.IntegerType);
+ }
+
+ @Test
+ public void testTsNtzToHours() {
+ ScalarFunction<Integer> tsNtzToHourFunc = new
HoursFunction.TimestampNtzToHoursFunction();
+ UserDefinedScalarFunc udf =
+ new UserDefinedScalarFunc(
+ tsNtzToHourFunc.name(),
+ tsNtzToHourFunc.canonicalName(),
+ expressions(FieldReference.apply("col1")));
+ testUDF(
+ udf,
+ Expressions.hour("col1"),
+ timestampNtzToHours("2023-12-03T10:15:30"),
+ DataTypes.IntegerType);
+ }
+
+ @Test
+ public void testBucket() {
+ ScalarFunction<Integer> bucketInt = new
BucketFunction.BucketInt(DataTypes.IntegerType);
+ UserDefinedScalarFunc udf =
+ new UserDefinedScalarFunc(
+ bucketInt.name(),
+ bucketInt.canonicalName(),
+ expressions(
+ LiteralValue.apply(4, DataTypes.IntegerType),
FieldReference.apply("col1")));
+ testUDF(udf, Expressions.bucket("col1", 4), 2, DataTypes.IntegerType);
+ }
+
+ @Test
+ public void testTruncate() {
+ ScalarFunction<UTF8String> truncate = new
TruncateFunction.TruncateString();
+ UserDefinedScalarFunc udf =
+ new UserDefinedScalarFunc(
+ truncate.name(),
+ truncate.canonicalName(),
+ expressions(
+ LiteralValue.apply(6, DataTypes.IntegerType),
FieldReference.apply("col1")));
+ testUDF(udf, Expressions.truncate("col1", 6), "prefix",
DataTypes.StringType);
+ }
+
+ @Test
+ public void testUnsupportedUDFConvert() {
+ ScalarFunction<UTF8String> icebergVersionFunc =
+ (ScalarFunction<UTF8String>) new IcebergVersionFunction().bind(new
StructType());
+ UserDefinedScalarFunc udf =
+ new UserDefinedScalarFunc(
+ icebergVersionFunc.name(),
+ icebergVersionFunc.canonicalName(),
+ new org.apache.spark.sql.connector.expressions.Expression[] {});
+ LiteralValue literalValue = new LiteralValue("1.3.0",
DataTypes.StringType);
+ Predicate predicate = new Predicate("=", expressions(udf, literalValue));
+
+ Expression icebergExpr = SparkV2Filters.convert(predicate);
+ Assertions.assertThat(icebergExpr).isNull();
+ }
+
+ private <T> void testUDF(
Review Comment:
I like how thorough this is. Thank you!
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
