This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new d6662001ae [vector] Fix partition filter handling in vector search
(#8366)
d6662001ae is described below
commit d6662001ae11f44fab17dec6a7cb2525cda1a8b9
Author: QuakeWang <[email protected]>
AuthorDate: Sat Jun 27 15:03:42 2026 +0800
[vector] Fix partition filter handling in vector search (#8366)
Fix partition filter handling in vector search builders.
Previously, `withFilter` extracted partition predicates but still kept
the full predicate in the scalar filter. For partition-only filters,
indexed vector reads could incorrectly enter the scalar pre-filter path
and return empty results when no scalar index files existed. In
addition, explicit `withPartitionFilter` calls and partition predicates
extracted from `withFilter` could overwrite each other instead of
behaving as a stable intersection.
---
.../table/source/BatchVectorSearchBuilderImpl.java | 39 +++++--
.../table/source/VectorSearchBuilderImpl.java | 39 +++++--
.../table/source/VectorSearchBuilderTest.java | 130 +++++++++++++++++++++
3 files changed, 192 insertions(+), 16 deletions(-)
diff --git
a/paimon-core/src/main/java/org/apache/paimon/table/source/BatchVectorSearchBuilderImpl.java
b/paimon-core/src/main/java/org/apache/paimon/table/source/BatchVectorSearchBuilderImpl.java
index 8a34691b7a..41716fc3d3 100644
---
a/paimon-core/src/main/java/org/apache/paimon/table/source/BatchVectorSearchBuilderImpl.java
+++
b/paimon-core/src/main/java/org/apache/paimon/table/source/BatchVectorSearchBuilderImpl.java
@@ -24,11 +24,15 @@ import org.apache.paimon.predicate.PredicateBuilder;
import org.apache.paimon.table.FileStoreTable;
import org.apache.paimon.table.InnerTable;
import org.apache.paimon.types.DataField;
+import org.apache.paimon.utils.Pair;
+import java.util.Arrays;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
+import java.util.Optional;
-import static
org.apache.paimon.partition.PartitionPredicate.splitPartitionPredicate;
+import static
org.apache.paimon.partition.PartitionPredicate.splitPartitionPredicatesAndDataPredicates;
import static org.apache.paimon.utils.Preconditions.checkArgument;
import static org.apache.paimon.utils.Preconditions.checkNotNull;
@@ -52,22 +56,41 @@ public class BatchVectorSearchBuilderImpl implements
BatchVectorSearchBuilder {
@Override
public BatchVectorSearchBuilder withPartitionFilter(PartitionPredicate
partitionFilter) {
- this.partitionFilter = partitionFilter;
+ addPartitionFilter(partitionFilter);
return this;
}
@Override
public BatchVectorSearchBuilder withFilter(Predicate predicate) {
- if (this.filter == null) {
- this.filter = predicate;
- } else {
- this.filter = PredicateBuilder.and(this.filter, predicate);
+ Pair<Optional<PartitionPredicate>, List<Predicate>> pair =
+ splitPartitionPredicatesAndDataPredicates(
+ predicate, table.rowType(), table.partitionKeys());
+ if (pair.getLeft().isPresent()) {
+ addPartitionFilter(pair.getLeft().get());
+ }
+ if (!pair.getRight().isEmpty()) {
+ Predicate dataFilter = PredicateBuilder.and(pair.getRight());
+ if (this.filter == null) {
+ this.filter = dataFilter;
+ } else {
+ this.filter = PredicateBuilder.and(this.filter, dataFilter);
+ }
}
- splitPartitionPredicate(predicate, table.rowType(),
table.partitionKeys())
- .ifPresent(value -> this.partitionFilter = value);
return this;
}
+ private void addPartitionFilter(PartitionPredicate partitionFilter) {
+ if (partitionFilter == null) {
+ return;
+ }
+ if (this.partitionFilter == null) {
+ this.partitionFilter = partitionFilter;
+ } else {
+ this.partitionFilter =
+ PartitionPredicate.and(Arrays.asList(this.partitionFilter,
partitionFilter));
+ }
+ }
+
@Override
public BatchVectorSearchBuilder withLimit(int limit) {
this.limit = limit;
diff --git
a/paimon-core/src/main/java/org/apache/paimon/table/source/VectorSearchBuilderImpl.java
b/paimon-core/src/main/java/org/apache/paimon/table/source/VectorSearchBuilderImpl.java
index 996f8b4f19..9b77c9c6a2 100644
---
a/paimon-core/src/main/java/org/apache/paimon/table/source/VectorSearchBuilderImpl.java
+++
b/paimon-core/src/main/java/org/apache/paimon/table/source/VectorSearchBuilderImpl.java
@@ -24,11 +24,15 @@ import org.apache.paimon.predicate.PredicateBuilder;
import org.apache.paimon.table.FileStoreTable;
import org.apache.paimon.table.InnerTable;
import org.apache.paimon.types.DataField;
+import org.apache.paimon.utils.Pair;
+import java.util.Arrays;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
+import java.util.Optional;
-import static
org.apache.paimon.partition.PartitionPredicate.splitPartitionPredicate;
+import static
org.apache.paimon.partition.PartitionPredicate.splitPartitionPredicatesAndDataPredicates;
import static org.apache.paimon.utils.Preconditions.checkNotNull;
/** Implementation for {@link VectorSearchBuilder}. */
@@ -51,22 +55,41 @@ public class VectorSearchBuilderImpl implements
VectorSearchBuilder {
@Override
public VectorSearchBuilder withPartitionFilter(PartitionPredicate
partitionFilter) {
- this.partitionFilter = partitionFilter;
+ addPartitionFilter(partitionFilter);
return this;
}
@Override
public VectorSearchBuilder withFilter(Predicate predicate) {
- if (this.filter == null) {
- this.filter = predicate;
- } else {
- this.filter = PredicateBuilder.and(this.filter, predicate);
+ Pair<Optional<PartitionPredicate>, List<Predicate>> pair =
+ splitPartitionPredicatesAndDataPredicates(
+ predicate, table.rowType(), table.partitionKeys());
+ if (pair.getLeft().isPresent()) {
+ addPartitionFilter(pair.getLeft().get());
+ }
+ if (!pair.getRight().isEmpty()) {
+ Predicate dataFilter = PredicateBuilder.and(pair.getRight());
+ if (this.filter == null) {
+ this.filter = dataFilter;
+ } else {
+ this.filter = PredicateBuilder.and(this.filter, dataFilter);
+ }
}
- splitPartitionPredicate(predicate, table.rowType(),
table.partitionKeys())
- .ifPresent(value -> this.partitionFilter = value);
return this;
}
+ private void addPartitionFilter(PartitionPredicate partitionFilter) {
+ if (partitionFilter == null) {
+ return;
+ }
+ if (this.partitionFilter == null) {
+ this.partitionFilter = partitionFilter;
+ } else {
+ this.partitionFilter =
+ PartitionPredicate.and(Arrays.asList(this.partitionFilter,
partitionFilter));
+ }
+ }
+
@Override
public VectorSearchBuilder withLimit(int limit) {
this.limit = limit;
diff --git
a/paimon-core/src/test/java/org/apache/paimon/table/source/VectorSearchBuilderTest.java
b/paimon-core/src/test/java/org/apache/paimon/table/source/VectorSearchBuilderTest.java
index b861ea29a4..b937efe223 100644
---
a/paimon-core/src/test/java/org/apache/paimon/table/source/VectorSearchBuilderTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/table/source/VectorSearchBuilderTest.java
@@ -679,6 +679,129 @@ public class VectorSearchBuilderTest extends
TableTestBase {
+ result2.results().getIntCardinality());
}
+ @Test
+ public void
testVectorSearchPartitionFilterAndExtractedFilterAreConjunctive() throws
Exception {
+ catalog.createTable(
+ identifier("partitioned_filter_table"),
+ withVectorSchemaOptions(
+ Schema.newBuilder()
+ .column("pt", DataTypes.INT())
+ .column("id", DataTypes.INT())
+ .column(VECTOR_FIELD_NAME, new
ArrayType(DataTypes.FLOAT()))
+ .partitionKeys("pt"))
+ .build(),
+ false);
+ FileStoreTable table =
getTable(identifier("partitioned_filter_table"));
+
+ float[][] pt1Vectors = {{1.0f, 0.0f}, {0.95f, 0.1f}};
+ float[][] pt2Vectors = {{0.0f, 1.0f}, {0.1f, 0.95f}};
+
+ writePartitionedVectors(table, 1, pt1Vectors);
+ writePartitionedVectors(table, 2, pt2Vectors);
+
+ RowType partitionType = RowType.of(DataTypes.INT());
+ InternalRowSerializer serializer = new
InternalRowSerializer(partitionType);
+ BinaryRow partition1 = serializer.toBinaryRow(GenericRow.of(1)).copy();
+ BinaryRow partition2 = serializer.toBinaryRow(GenericRow.of(2)).copy();
+
+ buildAndCommitPartitionedIndex(table, pt1Vectors, partition1, new
Range(0, 1));
+ buildAndCommitPartitionedIndex(table, pt2Vectors, partition2, new
Range(2, 3));
+
+ PartitionPredicate partitionFilter =
+ PartitionPredicate.fromMultiple(
+ partitionType, Collections.singletonList(partition1));
+ Predicate matchingPartitionFilter = new
PredicateBuilder(table.rowType()).equal(0, 1);
+ Predicate extractedPartitionFilter = new
PredicateBuilder(table.rowType()).equal(0, 2);
+
+ VectorScan.Plan plan =
+ table.newVectorSearchBuilder()
+ .withPartitionFilter(partitionFilter)
+ .withFilter(extractedPartitionFilter)
+ .withVector(new float[] {1.0f, 0.0f})
+ .withLimit(2)
+ .withVectorColumn(VECTOR_FIELD_NAME)
+ .newVectorScan()
+ .scan();
+
+ assertThat(plan.splits()).isEmpty();
+
+ VectorScan.Plan reverseOrderPlan =
+ table.newVectorSearchBuilder()
+ .withFilter(extractedPartitionFilter)
+ .withPartitionFilter(partitionFilter)
+ .withVector(new float[] {1.0f, 0.0f})
+ .withLimit(2)
+ .withVectorColumn(VECTOR_FIELD_NAME)
+ .newVectorScan()
+ .scan();
+
+ assertThat(reverseOrderPlan.splits()).isEmpty();
+
+ VectorScan.Plan batchPlan =
+ table.newBatchVectorSearchBuilder()
+ .withPartitionFilter(partitionFilter)
+ .withFilter(extractedPartitionFilter)
+ .withVectors(new float[][] {new float[] {1.0f, 0.0f}})
+ .withLimit(2)
+ .withVectorColumn(VECTOR_FIELD_NAME)
+ .newVectorScan()
+ .scan();
+
+ assertThat(batchPlan.splits()).isEmpty();
+
+ VectorScan.Plan reverseOrderBatchPlan =
+ table.newBatchVectorSearchBuilder()
+ .withFilter(extractedPartitionFilter)
+ .withPartitionFilter(partitionFilter)
+ .withVectors(new float[][] {new float[] {1.0f, 0.0f}})
+ .withLimit(2)
+ .withVectorColumn(VECTOR_FIELD_NAME)
+ .newVectorScan()
+ .scan();
+
+ assertThat(reverseOrderBatchPlan.splits()).isEmpty();
+
+ GlobalIndexResult matchingResult =
+ table.newVectorSearchBuilder()
+ .withFilter(matchingPartitionFilter)
+ .withVector(new float[] {1.0f, 0.0f})
+ .withLimit(2)
+ .withVectorColumn(VECTOR_FIELD_NAME)
+ .executeLocal();
+ assertResultRowsBetween(matchingResult, 0, 1);
+
+ GlobalIndexResult matchingResultWithPartitionFilter =
+ table.newVectorSearchBuilder()
+ .withPartitionFilter(partitionFilter)
+ .withFilter(matchingPartitionFilter)
+ .withVector(new float[] {1.0f, 0.0f})
+ .withLimit(2)
+ .withVectorColumn(VECTOR_FIELD_NAME)
+ .executeLocal();
+ assertResultRowsBetween(matchingResultWithPartitionFilter, 0, 1);
+
+ List<GlobalIndexResult> matchingBatchResults =
+ table.newBatchVectorSearchBuilder()
+ .withFilter(matchingPartitionFilter)
+ .withVectors(new float[][] {new float[] {1.0f, 0.0f}})
+ .withLimit(2)
+ .withVectorColumn(VECTOR_FIELD_NAME)
+ .executeBatchLocal();
+ assertThat(matchingBatchResults).hasSize(1);
+ assertResultRowsBetween(matchingBatchResults.get(0), 0, 1);
+
+ List<GlobalIndexResult> matchingBatchResultsWithPartitionFilter =
+ table.newBatchVectorSearchBuilder()
+ .withPartitionFilter(partitionFilter)
+ .withFilter(matchingPartitionFilter)
+ .withVectors(new float[][] {new float[] {1.0f, 0.0f}})
+ .withLimit(2)
+ .withVectorColumn(VECTOR_FIELD_NAME)
+ .executeBatchLocal();
+ assertThat(matchingBatchResultsWithPartitionFilter).hasSize(1);
+
assertResultRowsBetween(matchingBatchResultsWithPartitionFilter.get(0), 0, 1);
+ }
+
@Test
public void testScanPartialRangeIntersection() throws Exception {
catalog.createTable(
@@ -1163,6 +1286,13 @@ public class VectorSearchBuilderTest extends
TableTestBase {
return ids;
}
+ private void assertResultRowsBetween(GlobalIndexResult result, long start,
long end) {
+ assertThat(result.results().isEmpty()).isFalse();
+ for (long rowId : result.results()) {
+ assertThat(rowId).isBetween(start, end);
+ }
+ }
+
private void buildAndCommitIndex(FileStoreTable table, float[][] vectors)
throws Exception {
buildAndCommitIndex(table, VECTOR_FIELD_NAME, vectors);
}
