ahshahid commented on code in PR #16570:
URL: https://github.com/apache/iceberg/pull/16570#discussion_r3384360620
##########
spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/source/SparkScan.java:
##########
@@ -384,4 +389,78 @@ protected long adjustSplitSize(List<? extends ScanTask>
tasks, long splitSize) {
return splitSize;
}
}
+
+ protected static String createOrderedExprString(Stream<Expression>
exprStream) {
+ return exprStream
+ .flatMap(x -> ExpressionVisitors.visit(x,
ExpressionFlattener.INSTANCE).stream())
+ .map(Spark3Util::describe)
+ .sorted()
+ .collect(Collectors.joining(", "));
+ }
+
+ private static class ExpressionFlattener
+ extends ExpressionVisitors.ExpressionVisitor<List<Expression>> {
+
+ private static final ExpressionFlattener INSTANCE = new
ExpressionFlattener();
+
+ private ExpressionFlattener() {}
+
+ @Override
+ public List<Expression> alwaysTrue() {
+ return List.of(Expressions.alwaysTrue());
+ }
+
+ @Override
+ public List<Expression> alwaysFalse() {
+ return List.of(Expressions.alwaysFalse());
+ }
+
+ @Override
+ public List<Expression> not(List<Expression> result) {
+ // since its a list of expressions created by And, if more than 1, so it
will already be
+ // sorted.
+ return List.of(Expressions.not(mergeExpressions(result)));
+ }
+
+ @Override
+ public List<Expression> and(List<Expression> leftResult, List<Expression>
rightResult) {
+ List<Expression> flattened = Lists.newArrayList(leftResult);
+ flattened.addAll(rightResult);
+ // sort the flattened stream back else otherwise the subtree may have
ordering issue, when
+ // calculating hashCode
+ return flattened.stream()
+ .map(expr -> Pair.of(Spark3Util.describe(expr), expr))
Review Comment:
and hashCode and equals get invoked a lot
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
