This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 045d8f0e2c [core] Limit vector live-row planning to indexed ranges
(#8586)
045d8f0e2c is described below
commit 045d8f0e2c77e0bc3a99522489700646e6d76d57
Author: QuakeWang <[email protected]>
AuthorDate: Wed Jul 15 08:13:16 2026 +0800
[core] Limit vector live-row planning to indexed ranges (#8586)
---
.../paimon/table/source/AbstractVectorRead.java | 11 +++--
.../table/source/GlobalIndexLiveRowFilter.java | 22 +++++++---
.../table/source/VectorSearchBuilderTest.java | 50 ++++++++++++++++++++++
3 files changed, 75 insertions(+), 8 deletions(-)
diff --git
a/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractVectorRead.java
b/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractVectorRead.java
index 2789799bd0..931d78389a 100644
---
a/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractVectorRead.java
+++
b/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractVectorRead.java
@@ -127,13 +127,18 @@ public abstract class AbstractVectorRead implements
Serializable {
}
protected List<RoaringNavigableMap64>
preFilters(List<IndexVectorSearchSplit> splits) {
- RoaringNavigableMap64 liveRows =
GlobalIndexLiveRowFilter.liveRows(table, partitionFilter);
+ List<Range> indexedRowRanges = new ArrayList<>(splits.size());
+ for (IndexVectorSearchSplit split : splits) {
+ indexedRowRanges.add(new Range(split.rowRangeStart(),
split.rowRangeEnd()));
+ }
+
+ RoaringNavigableMap64 liveRows =
+ GlobalIndexLiveRowFilter.liveRows(table, partitionFilter,
indexedRowRanges);
RoaringNavigableMap64 matchedRows = scalarMatchedRows(splits);
List<RoaringNavigableMap64> includeRowIds = new
ArrayList<>(splits.size());
boolean hasFilter = false;
- for (IndexVectorSearchSplit split : splits) {
- Range splitRange = new Range(split.rowRangeStart(),
split.rowRangeEnd());
+ for (Range splitRange : indexedRowRanges) {
RoaringNavigableMap64 splitRows = bitmapOf(splitRange);
RoaringNavigableMap64 include = new RoaringNavigableMap64();
diff --git
a/paimon-core/src/main/java/org/apache/paimon/table/source/GlobalIndexLiveRowFilter.java
b/paimon-core/src/main/java/org/apache/paimon/table/source/GlobalIndexLiveRowFilter.java
index 63a33479ad..6539629251 100644
---
a/paimon-core/src/main/java/org/apache/paimon/table/source/GlobalIndexLiveRowFilter.java
+++
b/paimon-core/src/main/java/org/apache/paimon/table/source/GlobalIndexLiveRowFilter.java
@@ -23,6 +23,7 @@ import org.apache.paimon.deletionvectors.DeletionVector;
import org.apache.paimon.io.DataFileMeta;
import org.apache.paimon.partition.PartitionPredicate;
import org.apache.paimon.table.FileStoreTable;
+import org.apache.paimon.table.source.snapshot.SnapshotReader;
import org.apache.paimon.table.source.snapshot.TimeTravelUtil;
import org.apache.paimon.utils.Range;
import org.apache.paimon.utils.RoaringNavigableMap64;
@@ -39,6 +40,14 @@ class GlobalIndexLiveRowFilter {
@Nullable
static RoaringNavigableMap64 liveRows(
@Nullable FileStoreTable table, @Nullable PartitionPredicate
partitionFilter) {
+ return liveRows(table, partitionFilter, null);
+ }
+
+ @Nullable
+ static RoaringNavigableMap64 liveRows(
+ @Nullable FileStoreTable table,
+ @Nullable PartitionPredicate partitionFilter,
+ @Nullable List<Range> rowRanges) {
if (table == null || !table.coreOptions().deletionVectorsEnabled()) {
return null;
}
@@ -48,14 +57,17 @@ class GlobalIndexLiveRowFilter {
return null;
}
- RoaringNavigableMap64 liveRows = new RoaringNavigableMap64();
- for (Split split :
+ SnapshotReader snapshotReader =
table.newSnapshotReader()
.withPartitionFilter(partitionFilter)
.withMode(ScanMode.ALL)
- .withSnapshot(snapshot)
- .read()
- .splits()) {
+ .withSnapshot(snapshot);
+ if (rowRanges != null) {
+ snapshotReader.withRowRanges(rowRanges);
+ }
+
+ RoaringNavigableMap64 liveRows = new RoaringNavigableMap64();
+ for (Split split : snapshotReader.read().splits()) {
if (split instanceof DataSplit) {
addLiveRows(table, liveRows, (DataSplit) split);
}
diff --git
a/paimon-core/src/test/java/org/apache/paimon/table/source/VectorSearchBuilderTest.java
b/paimon-core/src/test/java/org/apache/paimon/table/source/VectorSearchBuilderTest.java
index 93e219f442..f704e1f370 100644
---
a/paimon-core/src/test/java/org/apache/paimon/table/source/VectorSearchBuilderTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/table/source/VectorSearchBuilderTest.java
@@ -24,6 +24,7 @@ import org.apache.paimon.data.GenericArray;
import org.apache.paimon.data.GenericRow;
import org.apache.paimon.data.InternalRow;
import org.apache.paimon.data.serializer.InternalRowSerializer;
+import org.apache.paimon.fs.Path;
import org.apache.paimon.globalindex.GlobalIndexBuilderUtils;
import org.apache.paimon.globalindex.GlobalIndexMultiColumnWriter;
import org.apache.paimon.globalindex.GlobalIndexResult;
@@ -220,6 +221,55 @@ public class VectorSearchBuilderTest extends TableTestBase
{
assertThat(readIds(table, result)).containsExactly(2, 3);
}
+ @Test
+ public void testVectorLiveRowPlanningSkipsUnindexedDeletionVectors()
throws Exception {
+ catalog.createTable(
+ identifier("vector_search_unindexed_deletion_vector"),
+ vectorSchemaBuilder(VECTOR_FIELD_NAME)
+ .option(CoreOptions.DELETION_VECTORS_ENABLED.key(),
"true")
+ .build(),
+ false);
+ FileStoreTable table =
getTable(identifier("vector_search_unindexed_deletion_vector"));
+
+ float[][] indexedVectors = {{0.0f, 0.0f}, {1.0f, 0.0f}};
+ writeVectors(table, indexedVectors);
+ writeVectors(table, new float[][] {{2.0f, 0.0f}, {3.0f, 0.0f}});
+ buildAndCommitVectorIndex(table, indexedVectors, new Range(0, 1));
+ commitDeletionVectors(table, 3L);
+
+ DeletionFile unindexedDeletionFile = null;
+ for (Split split : table.newSnapshotReader().read().splits()) {
+ if (!(split instanceof DataSplit)) {
+ continue;
+ }
+ DataSplit dataSplit = (DataSplit) split;
+ List<DeletionFile> deletionFiles =
dataSplit.deletionFiles().orElse(null);
+ if (deletionFiles == null) {
+ continue;
+ }
+ for (int i = 0; i < dataSplit.dataFiles().size(); i++) {
+ if (dataSplit
+ .dataFiles()
+ .get(i)
+ .nonNullRowIdRange()
+ .hasIntersection(new Range(3, 3))) {
+ unindexedDeletionFile = deletionFiles.get(i);
+ }
+ }
+ }
+ assertThat(unindexedDeletionFile).isNotNull();
+ assertThat(table.fileIO().delete(new
Path(unindexedDeletionFile.path()), false)).isTrue();
+
+ GlobalIndexResult result =
+ table.newVectorSearchBuilder()
+ .withVector(new float[] {0.0f, 0.0f})
+ .withLimit(2)
+ .withVectorColumn(VECTOR_FIELD_NAME)
+ .executeLocal();
+
+ assertThat(result.results()).containsExactly(0L, 1L);
+ }
+
@Test
public void testBatchVectorSearchExcludesDeletedIndexedRows() throws
Exception {
catalog.createTable(
