This is an automated email from the ASF dual-hosted git repository.
XiaoHongbo-Hope pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 6c9c057267 [core][python] Default scalar-index.search-mode to fast
(#8891)
6c9c057267 is described below
commit 6c9c057267aa5ab215edda0ec1ef744e26ad7950
Author: XiaoHongbo <[email protected]>
AuthorDate: Wed Jul 29 11:42:37 2026 +0800
[core][python] Default scalar-index.search-mode to fast (#8891)
Align the scalar index query default with the vector and full-text index
families, which already default to 'fast'. Previously scalar index
queries defaulted to 'full', triggering an extra scan of unindexed row
ranges on every scalar predicate read.
### Purpose
### Tests
---
docs/docs/multimodal-table/global-index.mdx | 2 +-
docs/generated/core_configuration.html | 2 +-
.../src/main/java/org/apache/paimon/CoreOptions.java | 2 +-
.../src/test/java/org/apache/paimon/CoreOptionsTest.java | 2 +-
.../apache/paimon/table/BtreeGlobalIndexTableTest.java | 15 ++++++++++-----
.../paimon/table/source/VectorSearchBuilderTest.java | 12 +++++++-----
paimon-python/pypaimon/common/options/core_options.py | 2 +-
.../pypaimon/tests/e2e/java_py_read_write_test.py | 4 +++-
.../tests/global_index_scalar_search_mode_test.py | 6 +++---
9 files changed, 28 insertions(+), 19 deletions(-)
diff --git a/docs/docs/multimodal-table/global-index.mdx
b/docs/docs/multimodal-table/global-index.mdx
index 731fb8c078..54aa978f1a 100644
--- a/docs/docs/multimodal-table/global-index.mdx
+++ b/docs/docs/multimodal-table/global-index.mdx
@@ -395,7 +395,7 @@ These table options affect global index build and read
behavior:
|---|---|---|
| `global-index.enabled` | `true` | Whether scans can use global indexes. |
| `global-index.search-mode` | Not set | Legacy fallback search mode for
global-index queries. Family-specific options take precedence. |
-| `scalar-index.search-mode` | `full` | Search mode for BTree and Bitmap
queries. |
+| `scalar-index.search-mode` | `fast` | Search mode for BTree and Bitmap
queries. |
| `vector-index.search-mode` | `fast` | Search mode for vector queries. |
| `full-text-index.search-mode` | `fast` | Search mode for full-text queries. |
| `global-index.external-path` | Not set | Root directory for global index
files. If not set, files are stored under the table index directory. |
diff --git a/docs/generated/core_configuration.html
b/docs/generated/core_configuration.html
index a4ea3baecd..6e80db093f 100644
--- a/docs/generated/core_configuration.html
+++ b/docs/generated/core_configuration.html
@@ -1364,7 +1364,7 @@ For an internal format table in a REST catalog, it also
makes the catalog own th
</tr>
<tr>
<td><h5>scalar-index.search-mode</h5></td>
- <td style="word-wrap: break-word;">full</td>
+ <td style="word-wrap: break-word;">fast</td>
<td><p>Enum</p></td>
<td>Search mode for scalar index queries.<br /><br />Possible
values:<ul><li>"fast": Only search indexed data.</li><li>"full": Use snapshot
next row id and global index coverage to detect missing row ids, and scan raw
data only when a gap exists.</li><li>"detail": Scan data files to find exact
unindexed rows. This can handle index invalidation caused by updates or
rewrites.</li></ul></td>
</tr>
diff --git a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java
b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java
index 92a62c344a..f3f22b6e7b 100644
--- a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java
+++ b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java
@@ -2796,7 +2796,7 @@ public class CoreOptions implements Serializable {
public static final ConfigOption<GlobalIndexSearchMode>
SCALAR_INDEX_SEARCH_MODE =
key("scalar-index.search-mode")
.enumType(GlobalIndexSearchMode.class)
- .defaultValue(GlobalIndexSearchMode.FULL)
+ .defaultValue(GlobalIndexSearchMode.FAST)
.withDescription("Search mode for scalar index queries.");
public static final ConfigOption<GlobalIndexSearchMode>
VECTOR_INDEX_SEARCH_MODE =
diff --git a/paimon-core/src/test/java/org/apache/paimon/CoreOptionsTest.java
b/paimon-core/src/test/java/org/apache/paimon/CoreOptionsTest.java
index 6f7fe93eef..1eefc0377c 100644
--- a/paimon-core/src/test/java/org/apache/paimon/CoreOptionsTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/CoreOptionsTest.java
@@ -113,7 +113,7 @@ public class CoreOptionsTest {
CoreOptions options = new CoreOptions(conf);
assertThat(options.globalIndexSearchMode()).isNull();
assertThat(options.scalarIndexSearchMode())
- .isEqualTo(CoreOptions.GlobalIndexSearchMode.FULL);
+ .isEqualTo(CoreOptions.GlobalIndexSearchMode.FAST);
assertThat(options.vectorIndexSearchMode())
.isEqualTo(CoreOptions.GlobalIndexSearchMode.FAST);
assertThat(options.fullTextIndexSearchMode())
diff --git
a/paimon-core/src/test/java/org/apache/paimon/table/BtreeGlobalIndexTableTest.java
b/paimon-core/src/test/java/org/apache/paimon/table/BtreeGlobalIndexTableTest.java
index 53f1ecc0e1..676a1c305e 100644
---
a/paimon-core/src/test/java/org/apache/paimon/table/BtreeGlobalIndexTableTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/table/BtreeGlobalIndexTableTest.java
@@ -208,7 +208,7 @@ public class BtreeGlobalIndexTableTest extends
DataEvolutionTestBase {
assertThat(logs)
.containsPattern(
"INFO Scan table '[^']+' with global index\\. "
- + "searchMode='full', total=\\d+ ms,
metadata=\\d+ ms, "
+ + "searchMode='fast', total=\\d+ ms,
metadata=\\d+ ms, "
+ "lookup=\\d+ ms, coverage=\\d+ ms\\.")
.containsPattern(
"INFO Global index lookup table='[^']+',
type='btree', "
@@ -238,7 +238,9 @@ public class BtreeGlobalIndexTableTest extends
DataEvolutionTestBase {
BinaryString.fromString("a100"),
BinaryString.fromString("a700")));
- assertThat(readF1(table, predicate)).containsExactly("a100", "a700");
+ // Default scalar-index.search-mode is 'fast': only indexed rows are
+ // returned, so the unindexed a700 is dropped.
+ assertThat(readF1(table, predicate)).containsExactly("a100");
assertThat(readF1(tableWithSearchMode(table, "fast"),
predicate)).containsExactly("a100");
assertThat(readF1(tableWithSearchMode(table, "full"), predicate))
@@ -252,7 +254,8 @@ public class BtreeGlobalIndexTableTest extends
DataEvolutionTestBase {
builder.equal(1, BinaryString.fromString("a700")),
builder.equal(2, BinaryString.fromString("b700")));
- assertThat(readF1(table,
andWithUnindexedField)).containsExactly("a700");
+ // Default 'fast': a700 lives in the unindexed range, so it is dropped.
+ assertThat(readF1(table, andWithUnindexedField)).isEmpty();
assertThat(readF1(tableWithSearchMode(table, "fast"),
andWithUnindexedField)).isEmpty();
assertThat(readF1(tableWithSearchMode(table, "full"),
andWithUnindexedField))
.containsExactly("a700");
@@ -365,7 +368,8 @@ public class BtreeGlobalIndexTableTest extends
DataEvolutionTestBase {
builder.equal(1, BinaryString.fromString("a700")),
builder.equal(2, BinaryString.fromString("b700")));
- assertThat(readF1(table, andPredicate)).containsExactly("a700");
+ // Default 'fast': a700 is in the unindexed range, so it is dropped.
+ assertThat(readF1(table, andPredicate)).isEmpty();
assertThat(readF1(tableWithSearchMode(table, "fast"),
andPredicate)).isEmpty();
assertThat(readF1(tableWithSearchMode(table, "full"), andPredicate))
.containsExactly("a700");
@@ -377,7 +381,8 @@ public class BtreeGlobalIndexTableTest extends
DataEvolutionTestBase {
builder.equal(1, BinaryString.fromString("a700")),
builder.equal(2, BinaryString.fromString("b701")));
- assertThat(readF1(table, orPredicate)).containsExactly("a700", "a701");
+ // Default 'fast': a700 is unindexed and dropped; a701 (f2 indexed)
stays.
+ assertThat(readF1(table, orPredicate)).containsExactly("a701");
assertThat(readF1(tableWithSearchMode(table, "fast"),
orPredicate)).containsExactly("a701");
assertThat(readF1(tableWithSearchMode(table, "full"), orPredicate))
.containsExactly("a700", "a701");
diff --git
a/paimon-core/src/test/java/org/apache/paimon/table/source/VectorSearchBuilderTest.java
b/paimon-core/src/test/java/org/apache/paimon/table/source/VectorSearchBuilderTest.java
index 9f747807c0..85528388c9 100644
---
a/paimon-core/src/test/java/org/apache/paimon/table/source/VectorSearchBuilderTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/table/source/VectorSearchBuilderTest.java
@@ -1065,12 +1065,14 @@ public class VectorSearchBuilderTest extends
TableTestBase {
}
@Test
- public void testPartialScalarPreFilterMustNotDropUnindexedScalarRows()
throws Exception {
+ public void testPartialScalarPreFilterDropsUnindexedRowsInFastMode()
throws Exception {
+ // Default fast scalar mode drops unindexed rows: the btree covers ids
+ // 3-7, so id>=8 (rows 8,9 unindexed) yields an empty pre-filter.
catalog.createTable(
- identifier("default_scalar_full_partial_index_table"),
+ identifier("default_scalar_fast_partial_index_table"),
vectorSchemaBuilder(VECTOR_FIELD_NAME).build(),
false);
- FileStoreTable table =
getTable(identifier("default_scalar_full_partial_index_table"));
+ FileStoreTable table =
getTable(identifier("default_scalar_fast_partial_index_table"));
float[][] vectors = new float[10][];
for (int i = 0; i < vectors.length; i++) {
@@ -1091,7 +1093,7 @@ public class VectorSearchBuilderTest extends
TableTestBase {
VectorScan.Plan vectorPlan = searchBuilder.newVectorScan().scan();
GlobalIndexResult result =
searchBuilder.newVectorRead().read(vectorPlan);
- assertThat(result.results()).contains(8L);
+ assertThat(result.results().isEmpty()).isTrue();
ReadBuilder readBuilder = table.newReadBuilder().withFilter(idFilter);
TableScan.Plan readPlan =
readBuilder.newScan().withGlobalIndexResult(result).plan();
@@ -1099,7 +1101,7 @@ public class VectorSearchBuilderTest extends
TableTestBase {
try (RecordReader<InternalRow> reader =
readBuilder.newRead().createReader(readPlan)) {
reader.forEachRemaining(row -> ids.add(row.getInt(0)));
}
- assertThat(ids).containsExactly(8);
+ assertThat(ids).isEmpty();
}
@Test
diff --git a/paimon-python/pypaimon/common/options/core_options.py
b/paimon-python/pypaimon/common/options/core_options.py
index 2b168ee09f..f53c3722b4 100644
--- a/paimon-python/pypaimon/common/options/core_options.py
+++ b/paimon-python/pypaimon/common/options/core_options.py
@@ -738,7 +738,7 @@ class CoreOptions:
SCALAR_INDEX_SEARCH_MODE: ConfigOption[GlobalIndexSearchMode] = (
ConfigOptions.key("scalar-index.search-mode")
.enum_type(GlobalIndexSearchMode)
- .default_value(GlobalIndexSearchMode.FULL)
+ .default_value(GlobalIndexSearchMode.FAST)
.with_description("Search mode for scalar index queries.")
)
diff --git a/paimon-python/pypaimon/tests/e2e/java_py_read_write_test.py
b/paimon-python/pypaimon/tests/e2e/java_py_read_write_test.py
index b9213bd898..89b9833f1e 100644
--- a/paimon-python/pypaimon/tests/e2e/java_py_read_write_test.py
+++ b/paimon-python/pypaimon/tests/e2e/java_py_read_write_test.py
@@ -544,7 +544,9 @@ class JavaPyReadWriteTest(unittest.TestCase):
fast_builder.new_scan().plan().splits())
self.assertEqual(0, fast_result.num_rows)
- read_builder = table.new_read_builder()
+ # full mode falls back to a raw scan for the unindexed k4 row
+ full_table = table.copy({'scalar-index.search-mode': 'full'})
+ read_builder = full_table.new_read_builder()
read_builder.with_filter(
read_builder.new_predicate_builder().equal('k', 'k4'))
actual = read_builder.new_read().to_arrow(
diff --git
a/paimon-python/pypaimon/tests/global_index_scalar_search_mode_test.py
b/paimon-python/pypaimon/tests/global_index_scalar_search_mode_test.py
index b900c52ca0..1985af3287 100644
--- a/paimon-python/pypaimon/tests/global_index_scalar_search_mode_test.py
+++ b/paimon-python/pypaimon/tests/global_index_scalar_search_mode_test.py
@@ -53,7 +53,7 @@ class ScalarGlobalIndexSearchModeTest(unittest.TestCase):
options = CoreOptions(Options.from_none())
self.assertIsNone(options.global_index_search_mode())
self.assertEqual(
- GlobalIndexSearchMode.FULL, options.scalar_index_search_mode())
+ GlobalIndexSearchMode.FAST, options.scalar_index_search_mode())
self.assertEqual(
GlobalIndexSearchMode.FAST, options.vector_index_search_mode())
self.assertEqual(
@@ -88,7 +88,7 @@ class ScalarGlobalIndexSearchModeTest(unittest.TestCase):
full = coverage.unindexed_ranges(1,
search_mode=GlobalIndexSearchMode.FULL)
self.assertEqual([(100, 199)], [(r.from_, r.to) for r in full])
self.assertEqual(
- [(100, 199)],
+ [],
[(r.from_, r.to) for r in coverage.unindexed_ranges(1)])
def test_scanner_applies_passed_scalar_mode(self):
@@ -102,7 +102,7 @@ class ScalarGlobalIndexSearchModeTest(unittest.TestCase):
coverage = _coverage(CoreOptions(Options.from_none()))
scanner = SimpleNamespace(_coverage=coverage, _fields=[1])
result = DataEvolutionGlobalIndexScanner.unindexed_rows(scanner, None)
- self.assertEqual([(100, 199)], _ranges(result))
+ self.assertEqual([], _ranges(result))
if __name__ == "__main__":
