This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 04282834bb [core] Fix vector scan matching multi-column filter indexes
(#8284)
04282834bb is described below
commit 04282834bb4962387e476f8f789739b98aa70aa9
Author: QuakeWang <[email protected]>
AuthorDate: Fri Jun 19 11:49:14 2026 +0800
[core] Fix vector scan matching multi-column filter indexes (#8284)
Vector search splits must keep the vector index selection tied to the
vector column as the primary index field, because vector/full-text
readers rebuild the search indexer from `indexFieldId`.
The scalar pre-filter path is different: a multi-column global index can
satisfy a filter field stored in `extraFieldIds`. Previously
`VectorScanImpl` only matched filter fields against `indexFieldId`, so
vector search missed usable scalar indexes when the filtered column was
an extra field.
This PR matches vector filter fields against all indexed fields while
keeping vector/full-text search columns restricted to primary index
fields, and adds regression coverage for that contract.
---
.../apache/paimon/table/source/VectorScanImpl.java | 27 +++++++++++-----
.../table/source/FullTextSearchBuilderTest.java | 36 ++++++++++++++++++++-
.../table/source/VectorSearchBuilderTest.java | 37 +++++++++++++++++++++-
3 files changed, 90 insertions(+), 10 deletions(-)
diff --git
a/paimon-core/src/main/java/org/apache/paimon/table/source/VectorScanImpl.java
b/paimon-core/src/main/java/org/apache/paimon/table/source/VectorScanImpl.java
index b59363a326..74ae2afe8d 100644
---
a/paimon-core/src/main/java/org/apache/paimon/table/source/VectorScanImpl.java
+++
b/paimon-core/src/main/java/org/apache/paimon/table/source/VectorScanImpl.java
@@ -81,16 +81,12 @@ public class VectorScanImpl implements VectorScan {
if (globalIndex == null) {
return false;
}
- int fieldId = globalIndex.indexFieldId();
- if (vectorColumn.id() == fieldId ||
filterFieldIds.contains(fieldId)) {
+ if (isPrimaryColumn(globalIndex, vectorColumn.id())) {
return true;
}
- int[] extras = globalIndex.extraFieldIds();
- if (extras != null) {
- for (int extra : extras) {
- if (filterFieldIds.contains(extra)) {
- return true;
- }
+ for (int filterFieldId : filterFieldIds) {
+ if (containsField(globalIndex, filterFieldId)) {
+ return true;
}
}
return false;
@@ -137,4 +133,19 @@ public class VectorScanImpl implements VectorScan {
private static boolean isPrimaryColumn(GlobalIndexMeta meta, int fieldId) {
return meta.indexFieldId() == fieldId;
}
+
+ private static boolean containsField(GlobalIndexMeta meta, int fieldId) {
+ if (meta.indexFieldId() == fieldId) {
+ return true;
+ }
+ int[] extraFieldIds = meta.extraFieldIds();
+ if (extraFieldIds != null) {
+ for (int extraFieldId : extraFieldIds) {
+ if (extraFieldId == fieldId) {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
}
diff --git
a/paimon-core/src/test/java/org/apache/paimon/table/source/FullTextSearchBuilderTest.java
b/paimon-core/src/test/java/org/apache/paimon/table/source/FullTextSearchBuilderTest.java
index 13f84d1b97..00eccd5dc9 100644
---
a/paimon-core/src/test/java/org/apache/paimon/table/source/FullTextSearchBuilderTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/table/source/FullTextSearchBuilderTest.java
@@ -51,6 +51,7 @@ import org.apache.paimon.utils.Range;
import org.junit.jupiter.api.Test;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
import java.util.List;
@@ -344,6 +345,30 @@ public class FullTextSearchBuilderTest extends
TableTestBase {
assertThat(ids).contains(0, 1, 2);
}
+ @Test
+ public void testFullTextSearchRequiresTextColumnAsPrimaryField() throws
Exception {
+ createTableDefault();
+ FileStoreTable table = getTableDefault();
+
+ String[] documents = {"Apache Paimon", "vector search"};
+ writeDocuments(table, documents);
+ buildAndCommitIndexWithFields(
+ table,
+ documents,
+ Arrays.asList(
+ table.rowType().getField("id"),
table.rowType().getField(TEXT_FIELD_NAME)));
+
+ FullTextSearchBuilder searchBuilder =
+ table.newFullTextSearchBuilder()
+ .withQueryText("Paimon")
+ .withLimit(2)
+ .withTextColumn(TEXT_FIELD_NAME);
+
+ FullTextScan.Plan plan = searchBuilder.newFullTextScan().scan();
+ assertThat(plan.splits()).isEmpty();
+ assertThat(searchBuilder.executeLocal().results().isEmpty()).isTrue();
+ }
+
// ====================== Helper methods ======================
private void writeDocuments(FileStoreTable table, String[] documents)
throws Exception {
@@ -358,6 +383,15 @@ public class FullTextSearchBuilderTest extends
TableTestBase {
}
private void buildAndCommitIndex(FileStoreTable table, String[] documents)
throws Exception {
+ buildAndCommitIndexWithFields(
+ table,
+ documents,
+
Collections.singletonList(table.rowType().getField(TEXT_FIELD_NAME)));
+ }
+
+ private void buildAndCommitIndexWithFields(
+ FileStoreTable table, String[] documents, List<DataField>
indexFields)
+ throws Exception {
Options options = table.coreOptions().toConfiguration();
DataField textField = table.rowType().getField(TEXT_FIELD_NAME);
@@ -380,7 +414,7 @@ public class FullTextSearchBuilderTest extends
TableTestBase {
table.store().pathFactory().globalIndexFileFactory(),
table.coreOptions(),
rowRange,
- textField.id(),
+ indexFields,
TestFullTextGlobalIndexerFactory.IDENTIFIER,
entries);
diff --git
a/paimon-core/src/test/java/org/apache/paimon/table/source/VectorSearchBuilderTest.java
b/paimon-core/src/test/java/org/apache/paimon/table/source/VectorSearchBuilderTest.java
index 9898f93594..522d9fa732 100644
---
a/paimon-core/src/test/java/org/apache/paimon/table/source/VectorSearchBuilderTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/table/source/VectorSearchBuilderTest.java
@@ -60,6 +60,7 @@ import java.io.ByteArrayOutputStream;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
import java.util.List;
@@ -519,6 +520,30 @@ public class VectorSearchBuilderTest extends TableTestBase
{
assertThat(result.results().isEmpty()).isTrue();
}
+ @Test
+ public void testVectorSearchRequiresVectorColumnAsPrimaryField() throws
Exception {
+ createTableDefault();
+ FileStoreTable table = getTableDefault();
+
+ float[][] vectors = {{1.0f, 0.0f}, {0.0f, 1.0f}};
+ writeVectors(table, vectors);
+ buildAndCommitVectorIndexWithFields(
+ table,
+ vectors,
+ new Range(0, 1),
+ Arrays.asList(table.rowType().getField("id"),
table.rowType().getField("vec")));
+
+ VectorSearchBuilder searchBuilder =
+ table.newVectorSearchBuilder()
+ .withVector(new float[] {1.0f, 0.0f})
+ .withLimit(2)
+ .withVectorColumn(VECTOR_FIELD_NAME);
+
+ VectorScan.Plan plan = searchBuilder.newVectorScan().scan();
+ assertThat(plan.splits()).isEmpty();
+ assertThat(searchBuilder.executeLocal().results().isEmpty()).isTrue();
+ }
+
@Test
public void testVectorSearchSplitSerialization() throws Exception {
createTableDefault();
@@ -957,6 +982,16 @@ public class VectorSearchBuilderTest extends TableTestBase
{
private void buildAndCommitVectorIndex(FileStoreTable table, float[][]
vectors, Range rowRange)
throws Exception {
+ buildAndCommitVectorIndexWithFields(
+ table,
+ vectors,
+ rowRange,
+
Collections.singletonList(table.rowType().getField(VECTOR_FIELD_NAME)));
+ }
+
+ private void buildAndCommitVectorIndexWithFields(
+ FileStoreTable table, float[][] vectors, Range rowRange,
List<DataField> indexFields)
+ throws Exception {
Options options = table.coreOptions().toConfiguration();
DataField vectorField = table.rowType().getField(VECTOR_FIELD_NAME);
@@ -978,7 +1013,7 @@ public class VectorSearchBuilderTest extends TableTestBase
{
table.store().pathFactory().globalIndexFileFactory(),
table.coreOptions(),
rowRange,
- vectorField.id(),
+ indexFields,
TestVectorGlobalIndexerFactory.IDENTIFIER,
entries);
