This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 073e4d991c [format] Sync ORC copy fixes for 1.9.8 (#8384)
073e4d991c is described below
commit 073e4d991c6219c532d34d057e12db17bd1c5aae
Author: Jingsong Lee <[email protected]>
AuthorDate: Tue Jun 30 10:46:00 2026 +0800
[format] Sync ORC copy fixes for 1.9.8 (#8384)
Sync selected Apache ORC 1.9.8 fixes into Paimon's copied
`org.apache.orc` classes after the ORC dependency bump. This keeps the
local zstd-jni customizations while avoiding stale predicate pushdown
and range-read behavior from the older copied sources.
---
.../java/org/apache/orc/impl/PhysicalFsWriter.java | 4 +-
.../java/org/apache/orc/impl/RecordReaderImpl.java | 16 ++++-
.../org/apache/orc/impl/RecordReaderUtils.java | 29 ++++++--
.../main/java/org/apache/orc/impl/WriterImpl.java | 2 +-
.../org/apache/orc/impl/RecordReaderImplTest.java | 80 ++++++++++++++++++++++
5 files changed, 120 insertions(+), 11 deletions(-)
diff --git
a/paimon-format/src/main/java/org/apache/orc/impl/PhysicalFsWriter.java
b/paimon-format/src/main/java/org/apache/orc/impl/PhysicalFsWriter.java
index aab908fc71..868c577bb9 100644
--- a/paimon-format/src/main/java/org/apache/orc/impl/PhysicalFsWriter.java
+++ b/paimon-format/src/main/java/org/apache/orc/impl/PhysicalFsWriter.java
@@ -110,7 +110,7 @@ public class PhysicalFsWriter implements PhysicalWriter {
opts,
encryption);
this.path = path;
- LOG.info(
+ LOG.debug(
"ORC writer created for path: {} with stripeSize: {}
blockSize: {}"
+ " compression: {}",
path,
@@ -539,7 +539,7 @@ public class PhysicalFsWriter implements PhysicalWriter {
// space in the block
if (length < blockSize && length > availBlockSpace && addBlockPadding)
{
byte[] pad = new byte[(int) Math.min(HDFS_BUFFER_SIZE,
availBlockSpace)];
- LOG.info("Padding ORC by {} bytes while merging", availBlockSpace);
+ LOG.debug("Padding ORC by {} bytes while merging",
availBlockSpace);
start += availBlockSpace;
while (availBlockSpace > 0) {
int writeLen = (int) Math.min(availBlockSpace, pad.length);
diff --git
a/paimon-format/src/main/java/org/apache/orc/impl/RecordReaderImpl.java
b/paimon-format/src/main/java/org/apache/orc/impl/RecordReaderImpl.java
index 4c821185d7..ab91bb1946 100644
--- a/paimon-format/src/main/java/org/apache/orc/impl/RecordReaderImpl.java
+++ b/paimon-format/src/main/java/org/apache/orc/impl/RecordReaderImpl.java
@@ -747,6 +747,12 @@ public class RecordReaderImpl implements RecordReader {
TypeDescription type,
boolean writerUsedProlepticGregorian,
boolean useUTCTimestamp) {
+ // When statsProto is EMPTY_COLUMN_STATISTICS, this column does not
actually provide
+ // statistics, so we cannot make any assumptions.
+ if (statsProto == EMPTY_COLUMN_STATISTICS) {
+ return SearchArgument.TruthValue.YES_NO_NULL;
+ }
+
ColumnStatistics cs =
ColumnStatisticsImpl.deserialize(
null, statsProto, writerUsedProlepticGregorian, true);
@@ -840,14 +846,22 @@ public class RecordReaderImpl implements RecordReader {
ValueRange range,
BloomFilter bloomFilter,
boolean useUTCTimestamp) {
+ // An invalid range means no value, including null, is written to this
column.
if (!range.isValid()) {
- return SearchArgument.TruthValue.YES_NO_NULL;
+ return SearchArgument.TruthValue.NO;
}
// if we didn't have any values, everything must have been null
if (!range.hasValues()) {
if (predicate.getOperator() == PredicateLeaf.Operator.IS_NULL) {
return SearchArgument.TruthValue.YES;
+ } else if (predicate.getOperator() ==
PredicateLeaf.Operator.NULL_SAFE_EQUALS) {
+ Object literal = predicate.getLiteral();
+ if (literal == null) {
+ return SearchArgument.TruthValue.YES;
+ } else {
+ return SearchArgument.TruthValue.NO;
+ }
} else {
return SearchArgument.TruthValue.NULL;
}
diff --git
a/paimon-format/src/main/java/org/apache/orc/impl/RecordReaderUtils.java
b/paimon-format/src/main/java/org/apache/orc/impl/RecordReaderUtils.java
index 0ceaadb449..5546117744 100644
--- a/paimon-format/src/main/java/org/apache/orc/impl/RecordReaderUtils.java
+++ b/paimon-format/src/main/java/org/apache/orc/impl/RecordReaderUtils.java
@@ -653,18 +653,18 @@ public class RecordReaderUtils {
}
static ChunkReader create(BufferChunk from, BufferChunk to) {
- long f = Integer.MAX_VALUE;
- long e = Integer.MIN_VALUE;
+ long f = Long.MAX_VALUE;
+ long e = Long.MIN_VALUE;
- long cf = Integer.MAX_VALUE;
- long ef = Integer.MIN_VALUE;
- int reqBytes = 0;
+ long cf = Long.MAX_VALUE;
+ long ef = Long.MIN_VALUE;
+ long reqBytes = 0L;
BufferChunk current = from;
while (current != to.next) {
f = Math.min(f, current.getOffset());
e = Math.max(e, current.getEnd());
- if (ef == Integer.MIN_VALUE || current.getOffset() <= ef) {
+ if (ef == Long.MIN_VALUE || current.getOffset() <= ef) {
cf = Math.min(cf, current.getOffset());
ef = Math.max(ef, current.getEnd());
} else {
@@ -675,7 +675,22 @@ public class RecordReaderUtils {
current = (BufferChunk) current.next;
}
reqBytes += ef - cf;
- return new ChunkReader(from, to, (int) (e - f), reqBytes);
+ if (reqBytes > IOUtils.MAX_ARRAY_SIZE) {
+ throw new IllegalArgumentException(
+ "invalid reqBytes value "
+ + reqBytes
+ + ",out of bounds "
+ + IOUtils.MAX_ARRAY_SIZE);
+ }
+ long readBytes = e - f;
+ if (readBytes > IOUtils.MAX_ARRAY_SIZE) {
+ throw new IllegalArgumentException(
+ "invalid readBytes value "
+ + readBytes
+ + ",out of bounds "
+ + IOUtils.MAX_ARRAY_SIZE);
+ }
+ return new ChunkReader(from, to, (int) readBytes, (int) reqBytes);
}
static ChunkReader create(BufferChunk from, int minSeekSize) {
diff --git a/paimon-format/src/main/java/org/apache/orc/impl/WriterImpl.java
b/paimon-format/src/main/java/org/apache/orc/impl/WriterImpl.java
index 4249c8bd35..c6b027bfa5 100644
--- a/paimon-format/src/main/java/org/apache/orc/impl/WriterImpl.java
+++ b/paimon-format/src/main/java/org/apache/orc/impl/WriterImpl.java
@@ -235,7 +235,7 @@ public class WriterImpl implements WriterInternal,
MemoryManager.Callback {
treeWriter = TreeWriter.Factory.create(schema, null, new
StreamFactory());
- LOG.info(
+ LOG.debug(
"ORC writer created for path: {} with stripeSize: {} options:
{}",
path,
stripeSize,
diff --git
a/paimon-format/src/test/java/org/apache/orc/impl/RecordReaderImplTest.java
b/paimon-format/src/test/java/org/apache/orc/impl/RecordReaderImplTest.java
new file mode 100644
index 0000000000..aff77b867e
--- /dev/null
+++ b/paimon-format/src/test/java/org/apache/orc/impl/RecordReaderImplTest.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.orc.impl;
+
+import org.apache.hadoop.hive.ql.io.sarg.PredicateLeaf;
+import org.apache.hadoop.hive.ql.io.sarg.SearchArgument;
+import org.apache.orc.TypeDescription;
+import org.junit.jupiter.api.Test;
+
+import java.util.Collections;
+import java.util.List;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/** Tests for {@link RecordReaderImpl}. */
+public class RecordReaderImplTest {
+
+ @Test
+ public void testNullSafeEqualsWithAllNullStatistics() {
+ ColumnStatisticsImpl stats =
ColumnStatisticsImpl.create(TypeDescription.createString());
+ stats.setNull();
+
+ assertThat(
+ RecordReaderImpl.evaluatePredicate(
+ stats, nullSafeEqualsPredicate("value"), null))
+ .isEqualTo(SearchArgument.TruthValue.NO);
+ assertThat(RecordReaderImpl.evaluatePredicate(stats,
nullSafeEqualsPredicate(null), null))
+ .isEqualTo(SearchArgument.TruthValue.YES);
+ }
+
+ private static PredicateLeaf nullSafeEqualsPredicate(Object literal) {
+ return new PredicateLeaf() {
+ @Override
+ public Operator getOperator() {
+ return Operator.NULL_SAFE_EQUALS;
+ }
+
+ @Override
+ public Type getType() {
+ return Type.STRING;
+ }
+
+ @Override
+ public String getColumnName() {
+ return "field";
+ }
+
+ @Override
+ public Object getLiteral() {
+ return literal;
+ }
+
+ @Override
+ public List<Object> getLiteralList() {
+ return Collections.emptyList();
+ }
+
+ @Override
+ public int getId() {
+ return 0;
+ }
+ };
+ }
+}
