This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new fde4de5566 [core] Exclude deletion-vector rows from primary-key ANN
builds (#8669)
fde4de5566 is described below
commit fde4de55664e4441f118ba76295ae74874baacbd
Author: Jingsong Lee <[email protected]>
AuthorDate: Wed Jul 15 23:54:13 2026 +0800
[core] Exclude deletion-vector rows from primary-key ANN builds (#8669)
---
.../designs/2026-07-15-pk-vector-dv-aware-build.md | 59 +++++++++++
.../pk/BucketedPrimaryKeyIndexMaintainer.java | 26 ++++-
.../pkvector/BucketedVectorIndexMaintainer.java | 82 +++++++++++++++-
.../index/pkvector/PkVectorAnnSegmentFile.java | 21 ++++
.../index/pkvector/PkVectorAnnSegmentSearcher.java | 3 +
.../paimon/operation/AbstractFileStoreWrite.java | 4 +-
.../BucketedVectorIndexMaintainerTest.java | 108 ++++++++++++++++++++-
.../index/pkvector/PkVectorAnnSegmentFileTest.java | 40 ++++++++
.../paimon/operation/PrimaryKeyIndexWriteTest.java | 13 ++-
9 files changed, 347 insertions(+), 9 deletions(-)
diff --git a/docs/designs/2026-07-15-pk-vector-dv-aware-build.md
b/docs/designs/2026-07-15-pk-vector-dv-aware-build.md
new file mode 100644
index 0000000000..fa8873a844
--- /dev/null
+++ b/docs/designs/2026-07-15-pk-vector-dv-aware-build.md
@@ -0,0 +1,59 @@
+# DV-aware Primary-key Vector Index Build Design
+
+## Problem Statement
+
+Primary-key vector index maintenance currently reads compact source files
without deletion-vector
+filtering and therefore indexes physical row positions that were already
deleted when an ANN build
+was scheduled. Query-time deletion-vector filtering preserves correctness, but
rebuilding the same
+active source cannot reclaim those holes.
+
+## Chosen Approach
+
+Capture the deletion vectors visible to the bucket writer when a pending ANN
build is created. Pass
+an immutable per-source snapshot into the existing `excludedPosition` hook
while preserving each
+vector's physical ordinal (`fileOffset + rowPosition`).
+
+## Design Details
+
+### Maintenance integration
+
+The bucket writer already restores and updates `BucketedDvMaintainer` before
primary-key index
+maintenance prepares its commit. Its deletion-vector factory will be passed to
the primary-key
+index maintainer and then to the vector maintainer.
+
+### Asynchronous snapshot semantics
+
+`PendingBuild` will clone relevant deletion vectors synchronously when the
build is scheduled. The
+executor must not observe later mutations of the writer's deletion-vector
state. Deletions created
+after scheduling remain covered by query-time filtering.
+
+### Row-id stability
+
+The vector reader continues to read every physical row without applying a
filtering reader.
+`PkVectorAnnSegmentFile` skips excluded vectors but writes included ids as
+`fileOffset + physicalRowPosition`, preserving source metadata and positional
reads.
+
+### Fully deleted sources
+
+If every source row is excluded, publish a zero-row segment backed by an empty
marker file. This
+keeps the source covered without passing deleted row ids to an ANN writer or
retrying the same
+build indefinitely. Search recognizes the zero-row segment and returns an
empty result without
+opening the marker as an ANN payload.
+
+### Tests
+
+1. A deletion present before scheduling is absent from the newly built ANN
payload.
+2. Mutating the writer deletion vector after scheduling does not change the
pending build snapshot.
+3. A fully deleted source publishes a searchable zero-row segment.
+4. Existing query-time deletion-vector filtering remains unchanged.
+
+## Open Questions
+
+None for this scoped change.
+
+## Out of Scope
+
+- Paimon vindex changes.
+- Stale asynchronous build discard/retry policy.
+- Source-file stale-ratio scheduling.
+- Removing query-time deletion-vector filtering.
diff --git
a/paimon-core/src/main/java/org/apache/paimon/index/pk/BucketedPrimaryKeyIndexMaintainer.java
b/paimon-core/src/main/java/org/apache/paimon/index/pk/BucketedPrimaryKeyIndexMaintainer.java
index dd3f244bb7..27faa7774f 100644
---
a/paimon-core/src/main/java/org/apache/paimon/index/pk/BucketedPrimaryKeyIndexMaintainer.java
+++
b/paimon-core/src/main/java/org/apache/paimon/index/pk/BucketedPrimaryKeyIndexMaintainer.java
@@ -20,6 +20,7 @@ package org.apache.paimon.index.pk;
import org.apache.paimon.CoreOptions;
import org.apache.paimon.data.BinaryRow;
+import org.apache.paimon.deletionvectors.DeletionVector;
import org.apache.paimon.disk.IOManager;
import org.apache.paimon.index.IndexFileHandler;
import org.apache.paimon.index.IndexFileMeta;
@@ -427,6 +428,24 @@ public final class BucketedPrimaryKeyIndexMaintainer {
@Nullable List<IndexFileMeta> restoredPayloads,
ExecutorService executor,
@Nullable IOManager ioManager) {
+ return create(
+ partition,
+ bucket,
+ restoredDataFiles,
+ restoredPayloads,
+ executor,
+ ioManager,
+ DeletionVector.emptyFactory());
+ }
+
+ public BucketedPrimaryKeyIndexMaintainer create(
+ BinaryRow partition,
+ int bucket,
+ @Nullable List<DataFileMeta> restoredDataFiles,
+ @Nullable List<IndexFileMeta> restoredPayloads,
+ ExecutorService executor,
+ @Nullable IOManager ioManager,
+ DeletionVector.Factory deletionVectorFactory) {
List<DataFileMeta> dataFiles =
restoredDataFiles == null ? Collections.emptyList() :
restoredDataFiles;
List<IndexFileMeta> payloads =
@@ -435,7 +454,12 @@ public final class BucketedPrimaryKeyIndexMaintainer {
vectorFactory == null
? null
: vectorFactory.create(
- partition, bucket, dataFiles, payloads,
executor);
+ partition,
+ bucket,
+ dataFiles,
+ payloads,
+ executor,
+ deletionVectorFactory);
BucketedFullTextIndexMaintainer fullText =
fullTextFactory == null
? null
diff --git
a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/BucketedVectorIndexMaintainer.java
b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/BucketedVectorIndexMaintainer.java
index 016b0d1320..014e7a3e34 100644
---
a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/BucketedVectorIndexMaintainer.java
+++
b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/BucketedVectorIndexMaintainer.java
@@ -20,6 +20,7 @@ package org.apache.paimon.index.pkvector;
import org.apache.paimon.CoreOptions;
import org.apache.paimon.data.BinaryRow;
+import org.apache.paimon.deletionvectors.DeletionVector;
import org.apache.paimon.index.IndexFileHandler;
import org.apache.paimon.index.IndexFileMeta;
import org.apache.paimon.index.pk.PrimaryKeyIndexLevels;
@@ -51,6 +52,7 @@ import java.util.concurrent.CancellationException;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
+import java.util.function.LongPredicate;
import static org.apache.paimon.utils.Preconditions.checkArgument;
@@ -64,6 +66,7 @@ public class BucketedVectorIndexMaintainer {
private final String metric;
private final String algorithm;
private final PkVectorDataFileReader.Factory vectorReaderFactory;
+ private final DeletionVector.Factory deletionVectorFactory;
private final PrimaryKeyIndexLevels<IndexFileMeta> annLevels;
private ExecutorService executor;
private final List<IndexFileMeta> annSegments;
@@ -81,6 +84,32 @@ public class BucketedVectorIndexMaintainer {
List<DataFileMeta> restoredDataFiles,
List<IndexFileMeta> restoredPayloads,
ExecutorService executor) {
+ this(
+ vectorFieldId,
+ annSegmentFile,
+ vectorField,
+ indexOptions,
+ metric,
+ algorithm,
+ vectorReaderFactory,
+ DeletionVector.emptyFactory(),
+ restoredDataFiles,
+ restoredPayloads,
+ executor);
+ }
+
+ BucketedVectorIndexMaintainer(
+ int vectorFieldId,
+ PkVectorAnnSegmentFile annSegmentFile,
+ DataField vectorField,
+ Options indexOptions,
+ String metric,
+ String algorithm,
+ PkVectorDataFileReader.Factory vectorReaderFactory,
+ DeletionVector.Factory deletionVectorFactory,
+ List<DataFileMeta> restoredDataFiles,
+ List<IndexFileMeta> restoredPayloads,
+ ExecutorService executor) {
this.vectorFieldId = vectorFieldId;
this.annSegmentFile = annSegmentFile;
this.vectorField = vectorField;
@@ -88,6 +117,7 @@ public class BucketedVectorIndexMaintainer {
this.metric = metric;
this.algorithm = algorithm;
this.vectorReaderFactory = vectorReaderFactory;
+ this.deletionVectorFactory = deletionVectorFactory;
CoreOptions coreOptions = new CoreOptions(indexOptions);
this.annLevels =
new PrimaryKeyIndexLevels<>(
@@ -215,7 +245,7 @@ public class BucketedVectorIndexMaintainer {
}
private void startPendingBuild(
- List<DataFileMeta> sourceFiles, List<IndexFileMeta> inputSegments)
{
+ List<DataFileMeta> sourceFiles, List<IndexFileMeta> inputSegments)
throws IOException {
PendingBuild build = new PendingBuild(sourceFiles, inputSegments);
build.start();
pendingBuild = build;
@@ -343,15 +373,21 @@ public class BucketedVectorIndexMaintainer {
}
}
- private IndexFileMeta buildAnnSegment(List<DataFileMeta> files) {
+ private IndexFileMeta buildAnnSegment(
+ List<DataFileMeta> files, Map<String, DeletionVector>
deletionVectors) {
try {
List<PkVectorAnnSegmentFile.Source> sources = new
ArrayList<>(files.size());
for (DataFileMeta file : files) {
PrimaryKeyIndexSourceFile sourceFile =
new PrimaryKeyIndexSourceFile(file.fileName(),
file.rowCount());
+ DeletionVector deletionVector =
deletionVectors.get(file.fileName());
+ LongPredicate excludedPosition =
+ deletionVector != null ? deletionVector::isDeleted :
position -> false;
sources.add(
PkVectorAnnSegmentFile.Source.lazy(
- sourceFile, () ->
vectorReaderFactory.create(file)));
+ sourceFile,
+ () -> vectorReaderFactory.create(file),
+ excludedPosition));
}
return annSegmentFile.build(sources, vectorField, indexOptions,
metric, algorithm);
} catch (IOException e) {
@@ -359,6 +395,21 @@ public class BucketedVectorIndexMaintainer {
}
}
+ private Map<String, DeletionVector>
snapshotDeletionVectors(List<DataFileMeta> files)
+ throws IOException {
+ Map<String, DeletionVector> snapshots = new LinkedHashMap<>();
+ for (DataFileMeta file : files) {
+ Optional<DeletionVector> deletionVector =
deletionVectorFactory.create(file.fileName());
+ if (deletionVector.isPresent() && !deletionVector.get().isEmpty())
{
+ snapshots.put(
+ file.fileName(),
+ DeletionVector.deserializeFromBytes(
+
DeletionVector.serializeToBytes(deletionVector.get())));
+ }
+ }
+ return snapshots;
+ }
+
private void validateCoverage(
List<IndexFileMeta> candidateAnn, Map<String, DataFileMeta>
sourceFiles) {
PkVectorBucketIndexState state =
@@ -392,20 +443,24 @@ public class BucketedVectorIndexMaintainer {
private final List<DataFileMeta> sourceFiles;
private final List<IndexFileMeta> inputSegments;
+ private final Map<String, DeletionVector> deletionVectors;
@Nullable private IndexFileMeta result;
@Nullable private Future<IndexFileMeta> future;
private boolean cancelled;
- private PendingBuild(List<DataFileMeta> sourceFiles,
List<IndexFileMeta> inputSegments) {
+ private PendingBuild(List<DataFileMeta> sourceFiles,
List<IndexFileMeta> inputSegments)
+ throws IOException {
this.sourceFiles = new ArrayList<>(sourceFiles);
this.inputSegments = new ArrayList<>(inputSegments);
+ this.deletionVectors = snapshotDeletionVectors(sourceFiles);
}
private void start() {
future =
executor.submit(
() -> {
- IndexFileMeta segment =
buildAnnSegment(sourceFiles);
+ IndexFileMeta segment =
+ buildAnnSegment(sourceFiles,
deletionVectors);
synchronized (PendingBuild.this) {
if (!cancelled) {
result = segment;
@@ -587,6 +642,22 @@ public class BucketedVectorIndexMaintainer {
@Nullable List<DataFileMeta> restoredDataFiles,
@Nullable List<IndexFileMeta> restoredPayloads,
ExecutorService executor) {
+ return create(
+ partition,
+ bucket,
+ restoredDataFiles,
+ restoredPayloads,
+ executor,
+ DeletionVector.emptyFactory());
+ }
+
+ public BucketedVectorIndexMaintainer create(
+ BinaryRow partition,
+ int bucket,
+ @Nullable List<DataFileMeta> restoredDataFiles,
+ @Nullable List<IndexFileMeta> restoredPayloads,
+ ExecutorService executor,
+ DeletionVector.Factory deletionVectorFactory) {
checkArgument(indexOptions != null, "ANN index options are not
configured.");
List<DataFileMeta> dataFiles =
restoredDataFiles == null ? Collections.emptyList() :
restoredDataFiles;
@@ -601,6 +672,7 @@ public class BucketedVectorIndexMaintainer {
algorithm,
new PkVectorDataFileReader.Factory(
readerFactoryBuilder, partition, bucket,
vectorField, vectorDimension),
+ deletionVectorFactory,
dataFiles,
payloads,
executor);
diff --git
a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFile.java
b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFile.java
index ed59ac9cf3..c466024f69 100644
---
a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFile.java
+++
b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFile.java
@@ -43,6 +43,7 @@ import javax.annotation.Nullable;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
@@ -142,6 +143,9 @@ public class PkVectorAnnSegmentFile extends IndexFile {
}
List<ResultEntry> results = writer.finish();
+ if (liveRowCount == 0 && results.isEmpty()) {
+ results = Collections.singletonList(fileWriter.emptyResult());
+ }
checkArgument(
results.size() == 1,
"ANN segment build must produce exactly one payload file,
but produced %s.",
@@ -205,6 +209,16 @@ public class PkVectorAnnSegmentFile extends IndexFile {
return path;
}
+ private ResultEntry emptyResult() throws IOException {
+ deleteCreatedFiles();
+ createdFiles.clear();
+ String fileName = newFileName("empty-vector");
+ try (PositionOutputStream ignored = newOutputStream(fileName)) {
+ // The searcher does not open payloads for segments without
live rows.
+ }
+ return new ResultEntry(fileName, 0, null);
+ }
+
private void deleteCreatedFiles() {
for (Path path : createdFiles.values()) {
fileIO.deleteQuietly(path);
@@ -253,6 +267,13 @@ public class PkVectorAnnSegmentFile extends IndexFile {
return new Source(sourceFile, readerFactory, position -> false);
}
+ static Source lazy(
+ PrimaryKeyIndexSourceFile sourceFile,
+ ReaderFactory readerFactory,
+ LongPredicate excludedPosition) {
+ return new Source(sourceFile, readerFactory, excludedPosition);
+ }
+
private PkVectorReader openReader() throws IOException {
return vectors != null ? vectors : readerFactory.open();
}
diff --git
a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentSearcher.java
b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentSearcher.java
index 58696e5a0b..788ff56121 100644
---
a/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentSearcher.java
+++
b/paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentSearcher.java
@@ -176,6 +176,9 @@ public class PkVectorAnnSegmentSearcher {
globalIndexMeta != null && globalIndexMeta.sourceMeta() !=
null,
"Vector segment %s has no source metadata.",
segment.fileName());
+ if (segment.rowCount() == 0) {
+ return Collections.emptyList();
+ }
GlobalIndexer indexer =
GlobalIndexer.create(segment.indexType(), vectorField,
indexOptions);
checkArgument(
diff --git
a/paimon-core/src/main/java/org/apache/paimon/operation/AbstractFileStoreWrite.java
b/paimon-core/src/main/java/org/apache/paimon/operation/AbstractFileStoreWrite.java
index a393613b35..214cfbb603 100644
---
a/paimon-core/src/main/java/org/apache/paimon/operation/AbstractFileStoreWrite.java
+++
b/paimon-core/src/main/java/org/apache/paimon/operation/AbstractFileStoreWrite.java
@@ -26,6 +26,7 @@ import org.apache.paimon.compact.CompactDeletionFile;
import org.apache.paimon.data.BinaryRow;
import org.apache.paimon.data.BlobConsumer;
import org.apache.paimon.deletionvectors.BucketedDvMaintainer;
+import org.apache.paimon.deletionvectors.DeletionVector;
import org.apache.paimon.disk.IOManager;
import org.apache.paimon.index.DynamicBucketIndexMaintainer;
import org.apache.paimon.index.IndexFileHandler;
@@ -517,7 +518,8 @@ public abstract class AbstractFileStoreWrite<T> implements
FileStoreWrite<T> {
restored.dataFiles(),
restored.sourceIndexPayloads(),
primaryKeyIndexExecutor(),
- ioManager);
+ ioManager,
+ DeletionVector.factory(dvMaintainer));
List<DataFileMeta> restoreFiles = restored.dataFiles();
if (restoreFiles == null) {
diff --git
a/paimon-core/src/test/java/org/apache/paimon/index/pkvector/BucketedVectorIndexMaintainerTest.java
b/paimon-core/src/test/java/org/apache/paimon/index/pkvector/BucketedVectorIndexMaintainerTest.java
index 35196debc0..4d06fb5fef 100644
---
a/paimon-core/src/test/java/org/apache/paimon/index/pkvector/BucketedVectorIndexMaintainerTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/index/pkvector/BucketedVectorIndexMaintainerTest.java
@@ -18,6 +18,7 @@
package org.apache.paimon.index.pkvector;
+import org.apache.paimon.deletionvectors.BitmapDeletionVector;
import org.apache.paimon.fs.Path;
import org.apache.paimon.fs.local.LocalFileIO;
import org.apache.paimon.index.GlobalIndexMeta;
@@ -44,6 +45,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
+import java.util.Optional;
import java.util.UUID;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
@@ -364,6 +366,106 @@ class BucketedVectorIndexMaintainerTest {
assertThat(commit.compactIncrement().get().newIndexFiles()).hasSize(1);
}
+ @Test
+ void testBuildExcludesDeletionVectorPositions() throws Exception {
+ LocalFileIO fileIO = LocalFileIO.create();
+ PkVectorAnnSegmentFile annFile = new PkVectorAnnSegmentFile(fileIO,
pathFactory());
+ DataField vectorField =
+ new DataField(7, "embedding", DataTypes.VECTOR(2,
DataTypes.FLOAT()));
+ DataFileMeta data = dataFile("data", 2);
+ PkVectorDataFileReader.Factory readerFactory =
mock(PkVectorDataFileReader.Factory.class);
+ PkVectorDataFileReader dataReader = reader(new float[][] {{1, 0}, {2,
0}});
+ when(readerFactory.create(data)).thenReturn(dataReader);
+ BitmapDeletionVector deletionVector = new BitmapDeletionVector();
+ deletionVector.delete(0);
+ BucketedVectorIndexMaintainer maintainer =
+ new BucketedVectorIndexMaintainer(
+ 7,
+ annFile,
+ vectorField,
+ indexOptions(),
+ "l2",
+ "test-vector-ann",
+ readerFactory,
+ fileName -> Optional.of(deletionVector),
+ Collections.emptyList(),
+ Collections.emptyList(),
+ executor);
+
+ BucketedVectorIndexMaintainer.VectorIndexCommit commit =
+ maintainer.prepareCommit(
+ DataIncrement.emptyIncrement(),
+ new CompactIncrement(
+ Collections.emptyList(),
+ Collections.singletonList(data),
+ Collections.emptyList()),
+ true);
+
+ assertThat(commit.compactIncrement()).isPresent();
+ assertThat(commit.compactIncrement().get().newIndexFiles())
+ .singleElement()
+ .extracting(IndexFileMeta::rowCount)
+ .isEqualTo(1L);
+ }
+
+ @Test
+ void testPendingBuildUsesDeletionVectorSnapshot() throws Exception {
+ LocalFileIO fileIO = LocalFileIO.create();
+ PkVectorAnnSegmentFile annFile = new PkVectorAnnSegmentFile(fileIO,
pathFactory());
+ DataField vectorField =
+ new DataField(7, "embedding", DataTypes.VECTOR(2,
DataTypes.FLOAT()));
+ DataFileMeta data = dataFile("data", 3);
+ PkVectorDataFileReader.Factory readerFactory =
mock(PkVectorDataFileReader.Factory.class);
+ PkVectorDataFileReader dataReader = reader(new float[][] {{1, 0}, {2,
0}, {3, 0}});
+ when(readerFactory.create(data)).thenReturn(dataReader);
+ BitmapDeletionVector deletionVector = new BitmapDeletionVector();
+ deletionVector.delete(0);
+ BucketedVectorIndexMaintainer maintainer =
+ new BucketedVectorIndexMaintainer(
+ 7,
+ annFile,
+ vectorField,
+ indexOptions(),
+ "l2",
+ "test-vector-ann",
+ readerFactory,
+ fileName -> Optional.of(deletionVector),
+ Collections.emptyList(),
+ Collections.emptyList(),
+ executor);
+ CountDownLatch executorBlocked = new CountDownLatch(1);
+ CountDownLatch releaseExecutor = new CountDownLatch(1);
+ executor.submit(
+ () -> {
+ executorBlocked.countDown();
+ releaseExecutor.await();
+ return null;
+ });
+ assertThat(executorBlocked.await(30, TimeUnit.SECONDS)).isTrue();
+
+ try {
+ maintainer.prepareCommit(
+ DataIncrement.emptyIncrement(),
+ new CompactIncrement(
+ Collections.emptyList(),
+ Collections.singletonList(data),
+ Collections.emptyList()),
+ false);
+ deletionVector.delete(1);
+ } finally {
+ releaseExecutor.countDown();
+ }
+
+ BucketedVectorIndexMaintainer.VectorIndexCommit commit =
+ maintainer.prepareCommit(
+ DataIncrement.emptyIncrement(),
CompactIncrement.emptyIncrement(), true);
+ assertThat(commit.appendIncrement()).isPresent();
+ assertThat(commit.appendIncrement().get().newIndexFiles())
+ .singleElement()
+ .extracting(IndexFileMeta::rowCount)
+ .isEqualTo(2L);
+ }
+
@Test
void testRebuildsDerivedLevelAndAtomicallyReplacesInputs() throws
Exception {
LocalFileIO fileIO = LocalFileIO.create();
@@ -708,10 +810,14 @@ class BucketedVectorIndexMaintainerTest {
}
private static DataFileMeta dataFile(String fileName) {
+ return dataFile(fileName, 1);
+ }
+
+ private static DataFileMeta dataFile(String fileName, long rowCount) {
return DataFileMeta.forAppend(
fileName,
100,
- 1,
+ rowCount,
SimpleStats.EMPTY_STATS,
0,
0,
diff --git
a/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFileTest.java
b/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFileTest.java
index a316af28a7..8e89ac6fc8 100644
---
a/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFileTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorAnnSegmentFileTest.java
@@ -80,6 +80,46 @@ class PkVectorAnnSegmentFileTest {
.containsExactly("data-1");
}
+ @Test
+ void testBuildsSearchableEmptySegmentWhenAllRowsAreExcluded() throws
Exception {
+ LocalFileIO fileIO = LocalFileIO.create();
+ PkVectorAnnSegmentFile annFile = annFile(fileIO);
+ IndexFileMeta segment =
+ annFile.build(
+ Collections.singletonList(
+ new PkVectorAnnSegmentFile.Source(
+ dataFile("data-1", 2),
+ new ArrayReader(new float[][] {{0, 0},
{1, 0}}),
+ position -> true)),
+ vectorField(),
+ indexOptions(),
+ "l2",
+ "test-vector-ann");
+
+ assertThat(segment.rowCount()).isZero();
+ ExecutorService executor = Executors.newSingleThreadExecutor();
+ try {
+ assertThat(
+ new PkVectorAnnSegmentSearcher(
+ fileIO,
+ annFile,
+ vectorField(),
+ indexOptions(),
+ "l2",
+ executor)
+ .search(
+ segment,
+
PrimaryKeyIndexSourceMeta.fromIndexFile(segment),
+ new float[] {0, 0},
+ 1,
+ Collections.emptyMap(),
+ Collections.emptyMap()))
+ .isEmpty();
+ } finally {
+ executor.shutdownNow();
+ }
+ }
+
@Test
void testBuildsAndSearchesMultiSourceSegment() throws Exception {
LocalFileIO fileIO = LocalFileIO.create();
diff --git
a/paimon-core/src/test/java/org/apache/paimon/operation/PrimaryKeyIndexWriteTest.java
b/paimon-core/src/test/java/org/apache/paimon/operation/PrimaryKeyIndexWriteTest.java
index ed16b21e01..0fd641f157 100644
---
a/paimon-core/src/test/java/org/apache/paimon/operation/PrimaryKeyIndexWriteTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/operation/PrimaryKeyIndexWriteTest.java
@@ -22,6 +22,8 @@ import org.apache.paimon.CoreOptions;
import org.apache.paimon.KeyValue;
import org.apache.paimon.TestFileStore;
import org.apache.paimon.TestKeyValueGenerator;
+import org.apache.paimon.deletionvectors.BitmapDeletionVector;
+import org.apache.paimon.deletionvectors.DeletionVector;
import org.apache.paimon.disk.IOManager;
import org.apache.paimon.disk.IOManagerImpl;
import org.apache.paimon.fs.Path;
@@ -122,9 +124,18 @@ class PrimaryKeyIndexWriteTest {
AbstractFileStoreWrite.WriterContainer<KeyValue> container =
write.createWriterContainer(generator.getPartition(record), 1);
- assertThat(readField(container.primaryKeyIndexMaintainer,
"vectorMaintainer")).isNotNull();
+ Object vectorMaintainer =
+ readField(container.primaryKeyIndexMaintainer,
"vectorMaintainer");
+ assertThat(vectorMaintainer).isNotNull();
assertThat((List<?>) readField(container.primaryKeyIndexMaintainer,
"sortedMaintainers"))
.hasSize(4);
+
+ BitmapDeletionVector deletionVector = new BitmapDeletionVector();
+ deletionVector.delete(1);
+ container.deletionVectorsMaintainer.deletionVectors().put("data-file",
deletionVector);
+ DeletionVector.Factory deletionVectorFactory =
+ (DeletionVector.Factory) readField(vectorMaintainer,
"deletionVectorFactory");
+
assertThat(deletionVectorFactory.create("data-file").get()).isSameAs(deletionVector);
write.close();
}
