This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new f2752953f6 [common] Stop answering value predicates from a truncated
BSI index (#9654)
f2752953f6 is described below
commit f2752953f6a737d8e895ad2f37efccd22e075804
Author: YangJie <[email protected]>
AuthorDate: Fri Sep 11 03:08:30 2026 -0400
[common] Stop answering value predicates from a truncated BSI index (#9654)
---
.../bsi/BitSliceIndexBitmapFileIndex.java | 44 ++++++++++++-
.../bsi/BitSliceIndexBitmapFileIndexTest.java | 64 ++++++++++++++++++
.../paimon/table/DataEvolutionFileIndexTest.java | 76 ++++++++++++++++++++++
3 files changed, 183 insertions(+), 1 deletion(-)
diff --git
a/paimon-common/src/main/java/org/apache/paimon/fileindex/bsi/BitSliceIndexBitmapFileIndex.java
b/paimon-common/src/main/java/org/apache/paimon/fileindex/bsi/BitSliceIndexBitmapFileIndex.java
index f9f0f95cec..e48e0934fc 100644
---
a/paimon-common/src/main/java/org/apache/paimon/fileindex/bsi/BitSliceIndexBitmapFileIndex.java
+++
b/paimon-common/src/main/java/org/apache/paimon/fileindex/bsi/BitSliceIndexBitmapFileIndex.java
@@ -95,12 +95,28 @@ public class BitSliceIndexBitmapFileIndex implements
FileIndexer {
? BitSliceIndexRoaringBitmap.map(input)
: BitSliceIndexRoaringBitmap.EMPTY;
- return new Reader(dataType, rowNumber, positive, negative);
+ Reader reader = new Reader(dataType, rowNumber, positive,
negative);
+ return valuesAreTruncated(dataType) ? new
TruncatedValueReader(reader) : reader;
} catch (Exception e) {
throw new RuntimeException(e);
}
}
+ /**
+ * Whether the value mapper loses information for this type. TIMESTAMP
above microsecond
+ * precision is mapped with {@link Timestamp#toMicros()}, so two values
that differ only below a
+ * microsecond share one indexed value.
+ */
+ private static boolean valuesAreTruncated(DataType dataType) {
+ if (dataType instanceof TimestampType) {
+ return ((TimestampType) dataType).getPrecision() > 6;
+ }
+ if (dataType instanceof LocalZonedTimestampType) {
+ return ((LocalZonedTimestampType) dataType).getPrecision() > 6;
+ }
+ return false;
+ }
+
private static class Writer extends FileIndexWriter {
private final Function<Object, Long> valueMapper;
@@ -358,6 +374,32 @@ public class BitSliceIndexBitmapFileIndex implements
FileIndexer {
}
}
+ /**
+ * Reader for a column whose values the mapper truncated, so comparing a
literal against the
+ * indexed value cannot answer the predicate: {@code ts <> '...000000000'}
would drop a row
+ * whose nanoseconds differ, and {@code ts = '...'} would select it.
Inheriting {@link
+ * FileIndexReader}'s {@code REMAIN} for those leaves the rows to be read
and filtered.
+ * Null-ness survives truncation, so those two questions still come from
the index.
+ */
+ private static class TruncatedValueReader extends FileIndexReader {
+
+ private final Reader reader;
+
+ public TruncatedValueReader(Reader reader) {
+ this.reader = reader;
+ }
+
+ @Override
+ public FileIndexResult visitIsNull(FieldRef fieldRef) {
+ return reader.visitIsNull(fieldRef);
+ }
+
+ @Override
+ public FileIndexResult visitIsNotNull(FieldRef fieldRef) {
+ return reader.visitIsNotNull(fieldRef);
+ }
+ }
+
public static Function<Object, Long> getValueMapper(DataType dataType) {
return dataType.accept(
new DataTypeDefaultVisitor<Function<Object, Long>>() {
diff --git
a/paimon-common/src/test/java/org/apache/paimon/fileindex/bsi/BitSliceIndexBitmapFileIndexTest.java
b/paimon-common/src/test/java/org/apache/paimon/fileindex/bsi/BitSliceIndexBitmapFileIndexTest.java
index ad60831ea2..467a569a15 100644
---
a/paimon-common/src/test/java/org/apache/paimon/fileindex/bsi/BitSliceIndexBitmapFileIndexTest.java
+++
b/paimon-common/src/test/java/org/apache/paimon/fileindex/bsi/BitSliceIndexBitmapFileIndexTest.java
@@ -18,13 +18,16 @@
package org.apache.paimon.fileindex.bsi;
+import org.apache.paimon.data.Timestamp;
import org.apache.paimon.fileindex.FileIndexReader;
+import org.apache.paimon.fileindex.FileIndexResult;
import org.apache.paimon.fileindex.FileIndexWriter;
import org.apache.paimon.fileindex.bitmap.BitmapIndexResult;
import org.apache.paimon.fs.ByteArraySeekableStream;
import org.apache.paimon.predicate.FieldRef;
import org.apache.paimon.types.BigIntType;
import org.apache.paimon.types.IntType;
+import org.apache.paimon.types.TimestampType;
import org.apache.paimon.utils.RoaringBitmap32;
import org.junit.jupiter.api.Test;
@@ -331,4 +334,65 @@ public class BitSliceIndexBitmapFileIndexTest {
.hasCauseInstanceOf(IllegalArgumentException.class)
.hasRootCauseMessage("values should be non-negative");
}
+
+ @Test
+ public void testSubMicrosecondTimestampIndexAnswersNoValuePredicate() {
+ // The value mapper stores micros, so these two rows share one indexed
value.
+ Timestamp second = Timestamp.fromEpochMillis(1000, 0);
+ Timestamp secondAndHalfMicro = Timestamp.fromEpochMillis(1000, 500);
+
+ TimestampType nanos = new TimestampType(9);
+ FieldRef fieldRef = new FieldRef(0, "", nanos);
+ BitSliceIndexBitmapFileIndex bsiFileIndex = new
BitSliceIndexBitmapFileIndex(nanos);
+ FileIndexWriter writer = bsiFileIndex.createWriter();
+ for (Object o : new Object[] {second, secondAndHalfMicro, null}) {
+ writer.write(o);
+ }
+ byte[] bytes = writer.serializedBytes();
+ FileIndexReader reader =
+ bsiFileIndex.createReader(new ByteArraySeekableStream(bytes),
0, bytes.length);
+
+ // Answering these from the index would drop row 1 from the <> result
and select it for
+ // the =, since the bitmap is the row set the scan reads.
+ assertThat(reader.visitEqual(fieldRef,
second)).isSameAs(FileIndexResult.REMAIN);
+ assertThat(reader.visitNotEqual(fieldRef,
second)).isSameAs(FileIndexResult.REMAIN);
+ assertThat(reader.visitIn(fieldRef, Arrays.asList(second,
secondAndHalfMicro)))
+ .isSameAs(FileIndexResult.REMAIN);
+ assertThat(reader.visitNotIn(fieldRef, Arrays.asList(second)))
+ .isSameAs(FileIndexResult.REMAIN);
+ assertThat(reader.visitLessThan(fieldRef, secondAndHalfMicro))
+ .isSameAs(FileIndexResult.REMAIN);
+ assertThat(reader.visitGreaterThan(fieldRef,
second)).isSameAs(FileIndexResult.REMAIN);
+ assertThat(reader.visitBetween(fieldRef, second, secondAndHalfMicro))
+ .isSameAs(FileIndexResult.REMAIN);
+
+ // Null-ness does not depend on the truncated digits, so it still
prunes.
+ assertThat(((BitmapIndexResult) reader.visitIsNull(fieldRef)).get())
+ .isEqualTo(RoaringBitmap32.bitmapOf(2));
+ assertThat(((BitmapIndexResult) reader.visitIsNotNull(fieldRef)).get())
+ .isEqualTo(RoaringBitmap32.bitmapOf(0, 1));
+ }
+
+ @Test
+ public void testMicrosecondTimestampIndexStillAnswersValuePredicates() {
+ // Precision 6 is exactly what the mapper stores, so nothing is given
up there.
+ Timestamp second = Timestamp.fromEpochMillis(1000, 0);
+ Timestamp secondAndMicro = Timestamp.fromEpochMillis(1000, 1000);
+
+ TimestampType micros = new TimestampType(6);
+ FieldRef fieldRef = new FieldRef(0, "", micros);
+ BitSliceIndexBitmapFileIndex bsiFileIndex = new
BitSliceIndexBitmapFileIndex(micros);
+ FileIndexWriter writer = bsiFileIndex.createWriter();
+ for (Object o : new Object[] {second, secondAndMicro}) {
+ writer.write(o);
+ }
+ byte[] bytes = writer.serializedBytes();
+ FileIndexReader reader =
+ bsiFileIndex.createReader(new ByteArraySeekableStream(bytes),
0, bytes.length);
+
+ assertThat(((BitmapIndexResult) reader.visitEqual(fieldRef,
second)).get())
+ .isEqualTo(RoaringBitmap32.bitmapOf(0));
+ assertThat(((BitmapIndexResult) reader.visitNotEqual(fieldRef,
second)).get())
+ .isEqualTo(RoaringBitmap32.bitmapOf(1));
+ }
}
diff --git
a/paimon-core/src/test/java/org/apache/paimon/table/DataEvolutionFileIndexTest.java
b/paimon-core/src/test/java/org/apache/paimon/table/DataEvolutionFileIndexTest.java
index 178e5ed9e2..a45a508057 100644
---
a/paimon-core/src/test/java/org/apache/paimon/table/DataEvolutionFileIndexTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/table/DataEvolutionFileIndexTest.java
@@ -24,6 +24,7 @@ import org.apache.paimon.data.BinaryRow;
import org.apache.paimon.data.BinaryString;
import org.apache.paimon.data.GenericRow;
import org.apache.paimon.data.InternalRow;
+import org.apache.paimon.data.Timestamp;
import org.apache.paimon.data.serializer.InternalRowSerializer;
import org.apache.paimon.deletionvectors.BitmapDeletionVector;
import org.apache.paimon.deletionvectors.DeletionVector;
@@ -31,6 +32,7 @@ import
org.apache.paimon.deletionvectors.append.BaseAppendDeleteFileMaintainer;
import org.apache.paimon.fileindex.FileIndexOptions;
import org.apache.paimon.fileindex.bitmap.BitmapFileIndexFactory;
import org.apache.paimon.fileindex.bloomfilter.BloomFilterFileIndexFactory;
+import org.apache.paimon.fileindex.bsi.BitSliceIndexBitmapFileIndexFactory;
import org.apache.paimon.index.IndexFileMeta;
import org.apache.paimon.io.CompactIncrement;
import org.apache.paimon.io.DataFileMeta;
@@ -557,6 +559,80 @@ public class DataEvolutionFileIndexTest extends
DataEvolutionTestBase {
return options;
}
+ @ParameterizedTest
+ @ValueSource(strings = {"parquet", "orc"})
+ public void testSubMicrosecondTimestampBsiMatchesUnindexed(String format)
throws Exception {
+ // A BSI index maps TIMESTAMP through toMicros(), so on a TIMESTAMP(9)
column two values in
+ // the same microsecond share one indexed value. Answering <>, strict
range or NOT IN from
+ // that bitmap drops rows the residual filter can no longer recover.
The fix makes the
+ // indexed read (executeFilter) return exactly what an unindexed full
scan does.
+ FileStoreTable table = createTimestampBsiTable("bsi_ts9_" + format,
format);
+
+ long base = 1_704_067_200_000L;
+ Timestamp tsA = Timestamp.fromEpochMillis(base, 123_000); // micro
bucket base*1000+123
+ Timestamp tsB = Timestamp.fromEpochMillis(base, 123_400); // same
bucket, but tsB > tsA
+ Timestamp tsC = Timestamp.fromEpochMillis(base, 999_000); // a
different bucket
+ write(
+ table,
+ GenericRow.of(0, tsA),
+ GenericRow.of(1, tsB),
+ GenericRow.of(2, tsC),
+ GenericRow.of(3, null));
+
+ PredicateBuilder b = new PredicateBuilder(table.rowType());
+
+ // Guard: the sub-microsecond nanos must survive the write/read round
trip on this format,
+ // otherwise tsA and tsB collapse and the comparison below would pass
vacuously.
+ List<Timestamp> stored = new ArrayList<>();
+ for (InternalRow row : fullScanFiltered(table, b.isNotNull(1))) {
+ stored.add(row.getTimestamp(1, 9));
+ }
+ assertThat(stored).contains(tsA, tsB);
+
+ // notEqual / strict lessThan / strict greaterThan are the ones that
lose rows to the
+ // micro-bucket collision; isNull/isNotNull still come from the index;
equal/between only
+ // over-select and are already corrected by the residual filter.
+ List<Predicate> predicates =
+ Arrays.asList(
+ b.notEqual(1, tsA),
+ b.lessThan(1, tsB),
+ b.greaterThan(1, tsA),
+ b.isNull(1),
+ b.isNotNull(1),
+ b.equal(1, tsA),
+ b.between(1, tsA, tsC));
+ for (Predicate p : predicates) {
+ assertThat(query(table, p))
+
.containsExactlyInAnyOrderElementsOf(fullScanFiltered(table, p));
+ }
+ }
+
+ private FileStoreTable createTimestampBsiTable(String name, String format)
throws Exception {
+ Schema.Builder builder =
+ Schema.newBuilder()
+ .column("f0", DataTypes.INT())
+ .column("f1", DataTypes.TIMESTAMP(9))
+ .option(CoreOptions.ROW_TRACKING_ENABLED.key(), "true")
+ .option(CoreOptions.DATA_EVOLUTION_ENABLED.key(),
"true")
+ .option(CoreOptions.FILE_FORMAT.key(), format);
+ bsiOptions("f1").forEach(builder::option);
+ Identifier identifier = identifier(name);
+ catalog.createTable(identifier, builder.build(), false);
+ return getTable(identifier);
+ }
+
+ private static Map<String, String> bsiOptions(String column) {
+ Map<String, String> options = new HashMap<>();
+ options.put(
+ FileIndexOptions.FILE_INDEX
+ + "."
+ + BitSliceIndexBitmapFileIndexFactory.BSI_INDEX
+ + "."
+ + CoreOptions.COLUMNS,
+ column);
+ return options;
+ }
+
private void writeAllColumns(FileStoreTable table, int count) throws
Exception {
BatchWriteBuilder builder = table.newBatchWriteBuilder();
try (BatchTableWrite write = builder.newWrite();
