This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new f88febf0c5 [vector][python] Read vindex vector indexes (#8293)
f88febf0c5 is described below
commit f88febf0c51a3e1ef187929f28b5d10b97484320
Author: Jingsong Lee <[email protected]>
AuthorDate: Sat Jun 20 10:00:36 2026 +0800
[vector][python] Read vindex vector indexes (#8293)
This PR flattens `paimon-vector` into a single jar module backed by
`paimon-vector-index-java` 0.1.0, removing the old `paimon-vector-jni`
submodule. It also adds Python read support for vindex vector indexes
through `paimon-vindex` 0.1.0 and covers Java-write/Python-read interop.
---
.github/workflows/utcase-vector-index.yml | 74 --------
.../lumina/index/LuminaVectorIndexOptions.java | 4 +-
paimon-python/dev/requirements-dev.txt | 2 +
paimon-python/dev/run_mixed_tests.sh | 79 +++++++-
.../pypaimon/globalindex/vindex/__init__.py | 18 ++
.../vindex/vindex_vector_global_index_reader.py | 198 ++++++++++++++++++++
.../pypaimon/table/source/vector_search_read.py | 8 +
.../pypaimon/tests/e2e/java_py_read_write_test.py | 33 ++++
.../pypaimon/tests/vindex_vector_index_test.py | 41 +++++
paimon-python/setup.py | 3 +
paimon-vector/paimon-vector-index/pom.xml | 200 ---------------------
paimon-vector/paimon-vector-jni/pom.xml | 69 -------
.../apache/paimon/index/vector/NativeLoader.java | 83 ---------
.../paimon/index/vector/VectorIndexInput.java | 23 ---
.../paimon/index/vector/VectorIndexMetadata.java | 94 ----------
.../paimon/index/vector/VectorIndexNative.java | 62 -------
.../paimon/index/vector/VectorIndexReader.java | 196 --------------------
.../paimon/index/vector/VectorIndexWriter.java | 131 --------------
.../index/vector/VectorSearchBatchResult.java | 100 -----------
.../paimon/index/vector/VectorSearchResult.java | 62 -------
paimon-vector/pom.xml | 175 +++++++++++++++++-
.../index/IvfFlatVectorGlobalIndexerFactory.java | 2 +-
.../IvfHnswFlatVectorGlobalIndexerFactory.java | 2 +-
.../index/IvfHnswSqVectorGlobalIndexerFactory.java | 2 +-
.../IvfPqAlgorithmVectorGlobalIndexerFactory.java | 2 +-
.../index/NativeVectorGlobalIndexReader.java} | 16 +-
.../index/NativeVectorGlobalIndexWriter.java} | 11 +-
.../vector/index/NativeVectorGlobalIndexer.java} | 11 +-
.../index/NativeVectorGlobalIndexerFactory.java} | 6 +-
.../vector/index/NativeVectorIndexLoader.java | 112 ++++++++++++
.../paimon/vector/index/VectorIndexMeta.java | 0
....apache.paimon.globalindex.GlobalIndexerFactory | 0
.../test/java/org/apache/paimon/JavaPyE2ETest.java | 200 +++++++++++++++++++++
.../vector/index/NativeVectorGlobalIndexTest.java} | 117 +++++++-----
.../NativeVectorGlobalIndexerFactoryTest.java} | 20 +--
.../index/SeekableStreamVectorIndexInputTest.java | 10 +-
36 files changed, 983 insertions(+), 1183 deletions(-)
diff --git a/.github/workflows/utcase-vector-index.yml
b/.github/workflows/utcase-vector-index.yml
deleted file mode 100644
index fa3da08567..0000000000
--- a/.github/workflows/utcase-vector-index.yml
+++ /dev/null
@@ -1,74 +0,0 @@
-################################################################################
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-################################################################################
-
-name: UTCase Vector Index
-
-on:
- push:
- paths:
- - 'paimon-vector/**'
- pull_request:
- paths:
- - 'paimon-vector/**'
-
-env:
- JDK_VERSION: 8
- MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=30
-Dmaven.wagon.http.retryHandler.requestSentEnabled=true
-
-concurrency:
- group: ${{ github.workflow }}-${{ github.event_name }}-${{
github.event.number || github.run_id }}
- cancel-in-progress: true
-
-jobs:
- vector_index_test:
- runs-on: ubuntu-latest
-
- steps:
- - name: Checkout code
- uses: actions/checkout@v6
-
- - name: Set up JDK ${{ env.JDK_VERSION }}
- uses: actions/setup-java@v5
- with:
- java-version: ${{ env.JDK_VERSION }}
- distribution: 'temurin'
-
- - name: Install Rust toolchain
- run: |
- curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s --
-y --default-toolchain stable --profile minimal
- echo "$HOME/.cargo/bin" >> $GITHUB_PATH
-
- - name: Clone and build paimon-vector-index native library
- run: |
- git clone --depth 1
https://github.com/apache/paimon-vector-index.git /tmp/paimon-vector-index
- cd /tmp/paimon-vector-index
- cargo build --release -p paimon-vindex-jni
-
- - name: Copy native library to resources
- run: |
-
RESOURCE_DIR=paimon-vector/paimon-vector-jni/src/main/resources/native/linux-amd64
- mkdir -p ${RESOURCE_DIR}
- cp /tmp/paimon-vector-index/target/release/libpaimon_vindex_jni.so
${RESOURCE_DIR}/
-
- - name: Build and test vector index modules
- timeout-minutes: 30
- run: |
- mvn -T 2C -B -ntp clean install -DskipTests
- mvn -B -ntp verify -pl
paimon-vector/paimon-vector-jni,paimon-vector/paimon-vector-index
-Dcheckstyle.skip=true -Dspotless.check.skip=true
- env:
- MAVEN_OPTS: -Xmx4096m
diff --git
a/paimon-lumina/src/main/java/org/apache/paimon/lumina/index/LuminaVectorIndexOptions.java
b/paimon-lumina/src/main/java/org/apache/paimon/lumina/index/LuminaVectorIndexOptions.java
index 843e5c3813..3c067ef93b 100644
---
a/paimon-lumina/src/main/java/org/apache/paimon/lumina/index/LuminaVectorIndexOptions.java
+++
b/paimon-lumina/src/main/java/org/apache/paimon/lumina/index/LuminaVectorIndexOptions.java
@@ -130,7 +130,7 @@ public class LuminaVectorIndexOptions {
/**
* Resolves per-field Lumina options for {@code fieldName} into an
effective {@link Options}.
*
- * <p>Following the convention shared with {@code paimon-vector-index} (PR
#8239), a field-level
+ * <p>Following the convention shared with {@code paimon-vector} (PR
#8239), a field-level
* option is written {@code fields.<fieldName>.<option>} — <b>without</b>
the {@code lumina.}
* index-type prefix — and overrides the column-agnostic {@code
lumina.<option>} for that field
* only. For example {@code fields.embed.distance.metric} overrides {@code
@@ -139,7 +139,7 @@ public class LuminaVectorIndexOptions {
*
* <p>Only recognized Lumina options (the keys in {@code
FIELD_OVERRIDABLE_KEYS}) are accepted;
* any other {@code fields.<fieldName>.*} key (e.g. a merge/aggregation
option) is left
- * untouched, mirroring how {@code paimon-vector-index} ignores keys it
does not recognize.
+ * untouched, mirroring how {@code paimon-vector} ignores keys it does not
recognize.
*
* <p>Each recognized field option is flattened back to its plain {@code
lumina.*} form, so the
* rest of this class still sees only {@code lumina.*} keys and the
metadata produced from these
diff --git a/paimon-python/dev/requirements-dev.txt
b/paimon-python/dev/requirements-dev.txt
index c83a2e44b8..c1c4e4eea9 100644
--- a/paimon-python/dev/requirements-dev.txt
+++ b/paimon-python/dev/requirements-dev.txt
@@ -32,3 +32,5 @@ vortex-data==0.70.0; python_version >= "3.11"
datafusion>=52; python_version >= "3.10"
# Lumina vector search (optional, for lumina index tests)
lumina-data>=0.1.0
+# paimon-vindex vector search (optional, for vindex index tests)
+paimon-vindex==0.1.0; python_version >= "3.9"
diff --git a/paimon-python/dev/run_mixed_tests.sh
b/paimon-python/dev/run_mixed_tests.sh
index b2c8aec7a4..c3d91893df 100755
--- a/paimon-python/dev/run_mixed_tests.sh
+++ b/paimon-python/dev/run_mixed_tests.sh
@@ -483,6 +483,64 @@ run_lumina_vector_btree_test() {
fi
}
+ensure_paimon_vindex() {
+ if python -c "import paimon_vindex" >/dev/null 2>&1; then
+ return 0
+ fi
+
+ echo "Installing Python paimon-vindex dependency..."
+ if python -m pip install 'paimon-vindex==0.1.0'; then
+ return 0
+ fi
+
+ echo -e "${YELLOW}Direct pip install failed; installing paimon-vindex into
a temporary target directory...${NC}"
+ local target_dir="${TMPDIR:-/tmp}/paimon-vindex-site"
+ rm -rf "$target_dir"
+ if python -m pip install --target "$target_dir" 'paimon-vindex==0.1.0';
then
+ export PYTHONPATH="$target_dir:${PYTHONPATH:-}"
+ return 0
+ fi
+
+ if python -c "import numpy" >/dev/null 2>&1; then
+ echo -e "${YELLOW}Dependency install failed but numpy is already
available; retrying paimon-vindex without dependencies...${NC}"
+ rm -rf "$target_dir"
+ if python -m pip install --target "$target_dir" --no-deps
'paimon-vindex==0.1.0'; then
+ export PYTHONPATH="$target_dir:${PYTHONPATH:-}"
+ return 0
+ fi
+ fi
+
+ echo -e "${RED}✗ Failed to install paimon-vindex${NC}"
+ return 1
+}
+
+# Function to run paimon-vindex vector index test (Java write index, Python
read and search)
+run_vindex_vector_test() {
+ echo -e "${YELLOW}=== Running paimon-vindex Vector Index Test (Java Write,
Python Read) ===${NC}"
+
+ cd "$PROJECT_ROOT"
+
+ echo "Running Maven test for JavaPyE2ETest.testVindexVectorIndexWrite..."
+ if mvn test
-Dtest=org.apache.paimon.JavaPyE2ETest#testVindexVectorIndexWrite -pl
paimon-vector -am -q -DfailIfNoTests=false -Drun.e2e.tests=true; then
+ echo -e "${GREEN}✓ Java test completed successfully${NC}"
+ else
+ echo -e "${RED}✗ Java test failed${NC}"
+ return 1
+ fi
+ cd "$PAIMON_PYTHON_DIR"
+ if ! ensure_paimon_vindex; then
+ return 1
+ fi
+ echo "Running Python test for
JavaPyReadWriteTest.test_read_vindex_vector_index..."
+ if python -m pytest
java_py_read_write_test.py::JavaPyReadWriteTest::test_read_vindex_vector_index
-v; then
+ echo -e "${GREEN}✓ Python test completed successfully${NC}"
+ return 0
+ else
+ echo -e "${RED}✗ Python test failed${NC}"
+ return 1
+ fi
+}
+
run_compact_conflict_test() {
echo -e "${YELLOW}=== Running Compact Conflict Test (Java Write Base,
Python Shard Update + Java Compact) ===${NC}"
@@ -745,6 +803,7 @@ main() {
local tantivy_fulltext_result=0
local lumina_vector_result=0
local lumina_vector_btree_result=0
+ local vindex_vector_result=0
local compact_conflict_result=0
local blob_compact_conflict_result=0
local blob_alter_compact_result=0
@@ -903,6 +962,18 @@ main() {
echo ""
+ # Run paimon-vindex vector index test (requires Python >= 3.9)
+ if [[ "$PYTHON_MINOR" -ge 9 ]]; then
+ if ! run_vindex_vector_test; then
+ vindex_vector_result=1
+ fi
+ else
+ echo -e "${YELLOW}⏭ Skipping paimon-vindex Vector Index Test (requires
Python >= 3.9, current: $PYTHON_VERSION)${NC}"
+ vindex_vector_result=0
+ fi
+
+ echo ""
+
# Run compact conflict test (Java write+compact, Python read)
if ! run_compact_conflict_test; then
compact_conflict_result=1
@@ -1068,6 +1139,12 @@ main() {
echo -e "${RED}✗ Lumina Vector + BTree Pre-Filter Test (Java Write,
Python Read): FAILED${NC}"
fi
+ if [[ $vindex_vector_result -eq 0 ]]; then
+ echo -e "${GREEN}✓ paimon-vindex Vector Index Test (Java Write, Python
Read): PASSED${NC}"
+ else
+ echo -e "${RED}✗ paimon-vindex Vector Index Test (Java Write, Python
Read): FAILED${NC}"
+ fi
+
if [[ $compact_conflict_result -eq 0 ]]; then
echo -e "${GREEN}✓ Compact Conflict Test (Java Write+Compact, Python
Read): PASSED${NC}"
else
@@ -1121,7 +1198,7 @@ main() {
# Clean up warehouse directory after all tests
cleanup_warehouse
- if [[ $java_write_result -eq 0 && $python_read_result -eq 0 &&
$python_write_result -eq 0 && $java_read_result -eq 0 && $pk_dv_result -eq 0 &&
$btree_index_result -eq 0 && $bitmap_index_result -eq 0 &&
$compressed_global_index_result -eq 0 && $compressed_text_result -eq 0 &&
$tantivy_fulltext_result -eq 0 && $lumina_vector_result -eq 0 &&
$lumina_vector_btree_result -eq 0 && $compact_conflict_result -eq 0 &&
$blob_compact_conflict_result -eq 0 && $blob_alter_compact_result -eq 0 && $
[...]
+ if [[ $java_write_result -eq 0 && $python_read_result -eq 0 &&
$python_write_result -eq 0 && $java_read_result -eq 0 && $pk_dv_result -eq 0 &&
$btree_index_result -eq 0 && $bitmap_index_result -eq 0 &&
$compressed_global_index_result -eq 0 && $compressed_text_result -eq 0 &&
$tantivy_fulltext_result -eq 0 && $lumina_vector_result -eq 0 &&
$lumina_vector_btree_result -eq 0 && $vindex_vector_result -eq 0 &&
$compact_conflict_result -eq 0 && $blob_compact_conflict_result -eq 0 && $blob_
[...]
echo -e "${GREEN}🎉 All tests passed! Java-Python interoperability
verified.${NC}"
return 0
else
diff --git a/paimon-python/pypaimon/globalindex/vindex/__init__.py
b/paimon-python/pypaimon/globalindex/vindex/__init__.py
new file mode 100644
index 0000000000..76373df86c
--- /dev/null
+++ b/paimon-python/pypaimon/globalindex/vindex/__init__.py
@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""paimon-vindex based global index readers."""
diff --git
a/paimon-python/pypaimon/globalindex/vindex/vindex_vector_global_index_reader.py
b/paimon-python/pypaimon/globalindex/vindex/vindex_vector_global_index_reader.py
new file mode 100644
index 0000000000..5b5c93a1b3
--- /dev/null
+++
b/paimon-python/pypaimon/globalindex/vindex/vindex_vector_global_index_reader.py
@@ -0,0 +1,198 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Vector global index reader using paimon-vindex."""
+
+import os
+import threading
+
+import numpy as np
+
+from pypaimon.common.file_io import pread, supports_pread
+from pypaimon.globalindex.global_index_reader import GlobalIndexReader,
_completed_future
+from pypaimon.globalindex.vector_search_result import
DictBasedScoredIndexResult
+
+VINDEX_IDENTIFIERS = ("ivf-flat", "ivf-pq", "ivf-hnsw-flat", "ivf-hnsw-sq")
+
+NPROBE_PARAMETER = "ivf.nprobe"
+EF_SEARCH_PARAMETER = "hnsw.ef_search"
+DEFAULT_NPROBE = 16
+DEFAULT_EF_SEARCH = 0
+
+
+class PaimonVindexInput:
+ """Input adapter required by paimon_vindex.VectorIndexReader."""
+
+ def __init__(self, stream):
+ self._stream = stream
+ self._supports_pread = supports_pread(stream)
+ self._lock = threading.Lock()
+
+ def pread_many(self, ranges):
+ if self._supports_pread:
+ return [pread(self._stream, length, offset) for offset, length in
ranges]
+
+ chunks = []
+ with self._lock:
+ for offset, length in ranges:
+ self._stream.seek(offset)
+ chunks.append(self._stream.read(length))
+ return chunks
+
+
+class VindexVectorGlobalIndexReader(GlobalIndexReader):
+ """Vector global index reader using paimon-vindex."""
+
+ def __init__(self, file_io, index_path, io_metas, options=None):
+ assert len(io_metas) == 1, "Expected exactly one index file per shard"
+ self._file_io = file_io
+ self._index_path = index_path
+ self._io_meta = io_metas[0]
+ self._options = dict(options or {})
+ self._stream = None
+ self._index_input = None
+ self._reader = None
+ self._metadata = None
+ self._load_lock = threading.Lock()
+
+ def visit_vector_search(self, vector_search):
+ self._ensure_loaded()
+
+ query = np.asarray(vector_search.vector, dtype=np.float32)
+ if query.ndim != 1:
+ raise ValueError("Query vector must be a one-dimensional float32
array")
+ expected_dim = self._metadata.dimension
+ if query.shape[0] != expected_dim:
+ raise ValueError(
+ "Query vector dimension mismatch: expected %d, got %d"
+ % (expected_dim, query.shape[0]))
+
+ effective_k = self._effective_k(vector_search)
+ if effective_k <= 0:
+ return _completed_future(None)
+
+ options = vector_search.options or {}
+ nprobe = _int_parameter(options, NPROBE_PARAMETER, DEFAULT_NPROBE)
+ ef_search = _int_parameter(options, EF_SEARCH_PARAMETER,
DEFAULT_EF_SEARCH)
+ filter_bytes = _filter_bytes(vector_search.include_row_ids)
+
+ ids, distances = self._reader.search(
+ query, effective_k, nprobe, ef_search, filter_bytes=filter_bytes)
+ id_to_scores = _build_scores(ids, distances, self._metadata.metric)
+ if not id_to_scores:
+ return _completed_future(None)
+ return _completed_future(DictBasedScoredIndexResult(id_to_scores))
+
+ def vector_metric(self):
+ self._ensure_loaded()
+ return self._metadata.metric
+
+ def _effective_k(self, vector_search):
+ limit = vector_search.limit
+ total_vectors = getattr(self._metadata, "total_vectors", limit)
+ effective_k = min(limit, int(total_vectors))
+ include_row_ids = vector_search.include_row_ids
+ if include_row_ids is not None:
+ cardinality = include_row_ids.cardinality()
+ if cardinality == 0:
+ return 0
+ effective_k = min(effective_k, cardinality)
+ return effective_k
+
+ def _ensure_loaded(self):
+ if self._reader is not None:
+ return
+
+ with self._load_lock:
+ if self._reader is not None:
+ return
+
+ try:
+ from paimon_vindex import VectorIndexReader
+ except ImportError as e:
+ raise ImportError(
+ "paimon-vindex is required to read vindex vector indexes. "
+ "Install paimon-vindex==0.1.0 or pypaimon[vindex].") from e
+
+ file_path = (self._io_meta.external_path
+ if self._io_meta.external_path
+ else os.path.join(self._index_path,
self._io_meta.file_name))
+ stream = self._file_io.new_input_stream(file_path)
+ try:
+ index_input = PaimonVindexInput(stream)
+ reader = VectorIndexReader(index_input)
+ self._metadata = reader.metadata()
+ self._index_input = index_input
+ self._reader = reader
+ self._stream = stream
+ except Exception:
+ stream.close()
+ raise
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ self.close()
+ return False
+
+ def close(self):
+ if self._reader is not None:
+ self._reader.close()
+ self._reader = None
+ if self._stream is not None:
+ self._stream.close()
+ self._stream = None
+
+
+def _filter_bytes(include_row_ids):
+ if include_row_ids is None:
+ return None
+ if include_row_ids.cardinality() == 0:
+ return None
+ return include_row_ids.serialize()
+
+
+def _build_scores(ids, distances, metric):
+ id_to_scores = {}
+ for row_id, distance in zip(ids, distances):
+ row_id = int(row_id)
+ if row_id < 0:
+ continue
+ id_to_scores[row_id] = _convert_distance_to_score(float(distance),
metric)
+ return id_to_scores
+
+
+def _convert_distance_to_score(distance, metric):
+ if metric == "l2":
+ return 1.0 / (1.0 + distance)
+ if metric == "cosine":
+ return 1.0 - distance
+ if metric == "inner_product":
+ return -distance
+ raise ValueError("Unknown vector search metric: %s" % metric)
+
+
+def _int_parameter(options, key, default_value):
+ value = options.get(key)
+ if value is None:
+ return default_value
+ try:
+ return int(value)
+ except ValueError as e:
+ raise ValueError(
+ "Invalid value for '%s': %s. Must be an integer." % (key, value))
from e
diff --git a/paimon-python/pypaimon/table/source/vector_search_read.py
b/paimon-python/pypaimon/table/source/vector_search_read.py
index 005a2ce85b..566b093cd6 100644
--- a/paimon-python/pypaimon/table/source/vector_search_read.py
+++ b/paimon-python/pypaimon/table/source/vector_search_read.py
@@ -225,8 +225,16 @@ def _create_vector_reader(index_type, file_io, index_path,
index_io_meta_list, o
LUMINA_IDENTIFIERS,
LuminaVectorGlobalIndexReader,
)
+ from pypaimon.globalindex.vindex.vindex_vector_global_index_reader import (
+ VINDEX_IDENTIFIERS,
+ VindexVectorGlobalIndexReader,
+ )
if index_type in LUMINA_IDENTIFIERS:
return LuminaVectorGlobalIndexReader(
file_io, index_path, index_io_meta_list, options
)
+ if index_type in VINDEX_IDENTIFIERS:
+ return VindexVectorGlobalIndexReader(
+ file_io, index_path, index_io_meta_list, options
+ )
raise ValueError("Unsupported vector index type: '%s'" % index_type)
diff --git a/paimon-python/pypaimon/tests/e2e/java_py_read_write_test.py
b/paimon-python/pypaimon/tests/e2e/java_py_read_write_test.py
index 5b2929cfac..f940f38a69 100644
--- a/paimon-python/pypaimon/tests/e2e/java_py_read_write_test.py
+++ b/paimon-python/pypaimon/tests/e2e/java_py_read_write_test.py
@@ -1220,6 +1220,39 @@ class JavaPyReadWriteTest(unittest.TestCase):
print(f"Lumina vector search ({label}) matched rows:
ids={ids}")
self.assertIn(0, ids)
+ def test_read_vindex_vector_index(self):
+ """Test reading a paimon-vindex vector index built by Java."""
+ if sys.version_info < (3, 9):
+ self.skipTest("paimon-vindex requires Python >= 3.9")
+ try:
+ import paimon_vindex # noqa: F401
+ except ImportError:
+ self.skipTest("paimon-vindex is not installed")
+
+ table = self.catalog.get_table('default.test_vindex_vector')
+
+ builder = table.new_vector_search_builder()
+ builder.with_vector_column('embedding')
+ builder.with_query_vector([1.0, 0.0, 0.0, 0.0])
+ builder.with_limit(3)
+
+ result = builder.execute_local()
+ row_ids = sorted(list(result.results()))
+ print(f"paimon-vindex vector search for [1,0,0,0]: row_ids={row_ids}")
+ self.assertIn(0, row_ids)
+ self.assertEqual(len(row_ids), 3)
+
+ read_builder = table.new_read_builder()
+ scan = read_builder.new_scan().with_global_index_result(result)
+ plan = scan.plan()
+ table_read = read_builder.new_read()
+ pa_table = table_read.to_arrow(plan.splits())
+ pa_table = table_sort_by(pa_table, 'id')
+ self.assertEqual(pa_table.num_rows, 3)
+ ids = pa_table.column('id').to_pylist()
+ print(f"paimon-vindex vector search matched rows: ids={ids}")
+ self.assertIn(0, ids)
+
def test_read_lumina_vector_with_btree_filter(self):
"""Vector search + btree scalar pre-filter, using a table that Java
populated with both a Lumina vector index on `embedding` and a BTree
diff --git a/paimon-python/pypaimon/tests/vindex_vector_index_test.py
b/paimon-python/pypaimon/tests/vindex_vector_index_test.py
new file mode 100644
index 0000000000..eae983ca42
--- /dev/null
+++ b/paimon-python/pypaimon/tests/vindex_vector_index_test.py
@@ -0,0 +1,41 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import unittest
+
+from pypaimon.globalindex.vector_search_result import
DictBasedScoredIndexResult
+from pypaimon.globalindex.vindex.vindex_vector_global_index_reader import
_build_scores
+
+
+class VindexVectorIndexTest(unittest.TestCase):
+
+ def test_inner_product_distance_converted_to_higher_is_better_score(self):
+ id_to_scores = _build_scores(
+ [10, 20, 30],
+ [-1.0, -0.5, -0.1],
+ "inner_product")
+
+ self.assertEqual(1.0, id_to_scores[10])
+ self.assertEqual(0.5, id_to_scores[20])
+ self.assertEqual(0.1, id_to_scores[30])
+
+ top1 = DictBasedScoredIndexResult(id_to_scores).top_k(1)
+ self.assertEqual([10], top1.results().to_list())
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/paimon-python/setup.py b/paimon-python/setup.py
index e4974bdc9b..c4a543f8fb 100644
--- a/paimon-python/setup.py
+++ b/paimon-python/setup.py
@@ -180,6 +180,9 @@ setup(
'lumina': [
'lumina-data>=0.1.0'
],
+ 'vindex': [
+ 'paimon-vindex==0.1.0; python_version>="3.9"',
+ ],
'sql': [
'pypaimon-rust; python_version>="3.10"',
'datafusion>=52; python_version>="3.10"',
diff --git a/paimon-vector/paimon-vector-index/pom.xml
b/paimon-vector/paimon-vector-index/pom.xml
deleted file mode 100644
index 6744de7c9e..0000000000
--- a/paimon-vector/paimon-vector-index/pom.xml
+++ /dev/null
@@ -1,200 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0";
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd";>
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <artifactId>paimon-vector</artifactId>
- <groupId>org.apache.paimon</groupId>
- <version>1.5-SNAPSHOT</version>
- </parent>
-
- <artifactId>paimon-vector-index</artifactId>
- <name>Paimon : Vector Index</name>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.paimon</groupId>
- <artifactId>paimon-vector-jni</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.paimon</groupId>
- <artifactId>paimon-common</artifactId>
- <version>${project.version}</version>
- <scope>provided</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.paimon</groupId>
- <artifactId>paimon-shade-jackson-2</artifactId>
-
<version>${paimon.shade.jackson.version}-${paimon.shade.version}</version>
- </dependency>
-
- <!-- test dependencies -->
- <dependency>
- <groupId>org.junit.jupiter</groupId>
- <artifactId>junit-jupiter</artifactId>
- <version>${junit5.version}</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.paimon</groupId>
- <artifactId>paimon-core</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.paimon</groupId>
- <artifactId>paimon-core</artifactId>
- <version>${project.version}</version>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.paimon</groupId>
- <artifactId>paimon-common</artifactId>
- <version>${project.version}</version>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.paimon</groupId>
- <artifactId>paimon-format</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.paimon</groupId>
- <artifactId>paimon-test-utils</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs-client</artifactId>
- <version>${hadoop.version}</version>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>org.apache.avro</groupId>
- <artifactId>avro</artifactId>
- </exclusion>
- <exclusion>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <version>${hadoop.version}</version>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>org.apache.avro</groupId>
- <artifactId>avro</artifactId>
- </exclusion>
- <exclusion>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- <exclusion>
- <groupId>jdk.tools</groupId>
- <artifactId>jdk.tools</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.protobuf</groupId>
- <artifactId>protobuf-java</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-mapreduce-client-core</artifactId>
- <version>${hadoop.version}</version>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>org.apache.avro</groupId>
- <artifactId>avro</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.protobuf</groupId>
- <artifactId>protobuf-java</artifactId>
- </exclusion>
- <exclusion>
- <groupId>ch.qos.reload4j</groupId>
- <artifactId>reload4j</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-reload4j</artifactId>
- </exclusion>
- <exclusion>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- <exclusion>
- <groupId>jdk.tools</groupId>
- <artifactId>jdk.tools</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- </dependencies>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-surefire-plugin</artifactId>
- <configuration>
- <forkCount>1</forkCount>
- <redirectTestOutputToFile>true</redirectTestOutputToFile>
- <parallel>none</parallel>
- </configuration>
- </plugin>
- </plugins>
- </build>
-</project>
diff --git a/paimon-vector/paimon-vector-jni/pom.xml
b/paimon-vector/paimon-vector-jni/pom.xml
deleted file mode 100644
index 23d00d043a..0000000000
--- a/paimon-vector/paimon-vector-jni/pom.xml
+++ /dev/null
@@ -1,69 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0";
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd";>
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <artifactId>paimon-vector</artifactId>
- <groupId>org.apache.paimon</groupId>
- <version>1.5-SNAPSHOT</version>
- </parent>
-
- <artifactId>paimon-vector-jni</artifactId>
- <name>Paimon : Vector Index JNI</name>
-
- <properties>
- <target.java.version>1.8</target.java.version>
- <spotless.check.skip>true</spotless.check.skip>
- <spotless.apply.skip>true</spotless.apply.skip>
- <checkstyle.skip>true</checkstyle.skip>
- </properties>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.paimon</groupId>
- <artifactId>paimon-shade-guava-30</artifactId>
-
<version>${paimon.shade.guava.version}-${paimon.shade.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.junit.jupiter</groupId>
- <artifactId>junit-jupiter</artifactId>
- <version>${junit5.version}</version>
- <scope>test</scope>
- </dependency>
- </dependencies>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-surefire-plugin</artifactId>
- <configuration>
- <forkCount>1</forkCount>
- <redirectTestOutputToFile>true</redirectTestOutputToFile>
- <parallel>none</parallel>
- </configuration>
- </plugin>
- </plugins>
- </build>
-</project>
diff --git
a/paimon-vector/paimon-vector-jni/src/main/java/org/apache/paimon/index/vector/NativeLoader.java
b/paimon-vector/paimon-vector-jni/src/main/java/org/apache/paimon/index/vector/NativeLoader.java
deleted file mode 100644
index e667bbcb8f..0000000000
---
a/paimon-vector/paimon-vector-jni/src/main/java/org/apache/paimon/index/vector/NativeLoader.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.paimon.index.vector;
-
-import org.apache.paimon.shade.guava30.com.google.common.io.ByteStreams;
-
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.util.Locale;
-
-/** Utility class for loading the native vector index JNI library. */
-public final class NativeLoader {
- private static boolean loaded = false;
-
- private NativeLoader() {}
-
- public static synchronized void loadJni() {
- if (loaded) {
- return;
- }
-
- String osName = System.getProperty("os.name").toLowerCase(Locale.ROOT);
- String osArch = System.getProperty("os.arch").toLowerCase(Locale.ROOT);
- String libName = "libpaimon_vindex_jni";
-
- String libExt;
- String osShortName;
- if (osName.contains("win")) {
- osShortName = "win";
- libExt = ".dll";
- libName += libExt;
- } else if (osName.contains("mac")) {
- osShortName = "darwin";
- libExt = ".dylib";
- libName += libExt;
- } else if (osName.contains("nix") || osName.contains("nux")) {
- osShortName = "linux";
- libExt = ".so";
- libName += libExt;
- } else {
- throw new UnsupportedOperationException("Unsupported OS: " +
osName);
- }
-
- String libPath = "/native/" + osShortName + "-" + osArch + "/" +
libName;
- try (InputStream in = NativeLoader.class.getResourceAsStream(libPath))
{
- if (in == null) {
- throw new FileNotFoundException("Library not found: " +
libPath);
- }
- File tempFile = File.createTempFile("libpaimon_vindex_jni",
libExt);
- tempFile.deleteOnExit();
-
- try (OutputStream out = new FileOutputStream(tempFile)) {
- ByteStreams.copy(in, out);
- }
- libName = tempFile.getAbsolutePath();
- } catch (IOException e) {
- throw new RuntimeException("Failed to load library: " +
e.getMessage(), e);
- }
-
- System.load(libName);
- loaded = true;
- }
-}
diff --git
a/paimon-vector/paimon-vector-jni/src/main/java/org/apache/paimon/index/vector/VectorIndexInput.java
b/paimon-vector/paimon-vector-jni/src/main/java/org/apache/paimon/index/vector/VectorIndexInput.java
deleted file mode 100644
index dca4430181..0000000000
---
a/paimon-vector/paimon-vector-jni/src/main/java/org/apache/paimon/index/vector/VectorIndexInput.java
+++ /dev/null
@@ -1,23 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.paimon.index.vector;
-
-public interface VectorIndexInput {
-
- void pread(long[] positions, byte[][] buffers);
-}
diff --git
a/paimon-vector/paimon-vector-jni/src/main/java/org/apache/paimon/index/vector/VectorIndexMetadata.java
b/paimon-vector/paimon-vector-jni/src/main/java/org/apache/paimon/index/vector/VectorIndexMetadata.java
deleted file mode 100644
index 4ffd89a4f3..0000000000
---
a/paimon-vector/paimon-vector-jni/src/main/java/org/apache/paimon/index/vector/VectorIndexMetadata.java
+++ /dev/null
@@ -1,94 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.paimon.index.vector;
-
-public final class VectorIndexMetadata {
-
- private final String indexType;
- private final int dimension;
- private final int nlist;
- private final String metric;
- private final long totalVectors;
- private final int pqM;
- private final int hnswM;
- private final int hnswEfConstruction;
- private final int hnswMaxLevel;
-
- public VectorIndexMetadata(
- String indexType,
- int dimension,
- int nlist,
- String metric,
- long totalVectors,
- int pqM,
- int hnswM,
- int efConstruction,
- int maxLevel) {
- if (indexType == null) {
- throw new NullPointerException("indexType");
- }
- if (metric == null) {
- throw new NullPointerException("metric");
- }
- this.indexType = indexType;
- this.dimension = dimension;
- this.nlist = nlist;
- this.metric = metric;
- this.totalVectors = totalVectors;
- this.pqM = pqM;
- this.hnswM = hnswM;
- this.hnswEfConstruction = efConstruction;
- this.hnswMaxLevel = maxLevel;
- }
-
- public String indexType() {
- return indexType;
- }
-
- public int dimension() {
- return dimension;
- }
-
- public int nlist() {
- return nlist;
- }
-
- public String metric() {
- return metric;
- }
-
- public long totalVectors() {
- return totalVectors;
- }
-
- public int pqM() {
- return pqM;
- }
-
- public int hnswM() {
- return hnswM;
- }
-
- public int hnswEfConstruction() {
- return hnswEfConstruction;
- }
-
- public int hnswMaxLevel() {
- return hnswMaxLevel;
- }
-}
diff --git
a/paimon-vector/paimon-vector-jni/src/main/java/org/apache/paimon/index/vector/VectorIndexNative.java
b/paimon-vector/paimon-vector-jni/src/main/java/org/apache/paimon/index/vector/VectorIndexNative.java
deleted file mode 100644
index b6a5bbf0a7..0000000000
---
a/paimon-vector/paimon-vector-jni/src/main/java/org/apache/paimon/index/vector/VectorIndexNative.java
+++ /dev/null
@@ -1,62 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.paimon.index.vector;
-
-final class VectorIndexNative {
-
- static {
- NativeLoader.loadJni();
- }
-
- private VectorIndexNative() {}
-
- static native long createWriter(String[] optionKeys, String[]
optionValues);
-
- static native int writerDimension(long ptr);
-
- static native void train(long ptr, float[] data, int n);
-
- static native void addVectors(long ptr, long[] ids, float[] data, int n);
-
- static native void writeIndex(long ptr, Object streamOutput);
-
- static native void freeWriter(long ptr);
-
- static native long openReader(Object streamInput);
-
- static native VectorIndexMetadata metadata(long ptr);
-
- static native VectorSearchResult search(long ptr, float[] query, int k,
int nprobe, int efSearch);
-
- static native VectorSearchResult searchWithRoaringFilter(
- long ptr, float[] query, int k, int nprobe, int efSearch, byte[]
roaringFilter);
-
- static native VectorSearchBatchResult searchBatch(
- long ptr, float[] queries, int queryCount, int k, int nprobe, int
efSearch);
-
- static native VectorSearchBatchResult searchBatchWithRoaringFilter(
- long ptr,
- float[] queries,
- int queryCount,
- int k,
- int nprobe,
- int efSearch,
- byte[] roaringFilter);
-
- static native void freeReader(long ptr);
-}
diff --git
a/paimon-vector/paimon-vector-jni/src/main/java/org/apache/paimon/index/vector/VectorIndexReader.java
b/paimon-vector/paimon-vector-jni/src/main/java/org/apache/paimon/index/vector/VectorIndexReader.java
deleted file mode 100644
index 34eefc7dca..0000000000
---
a/paimon-vector/paimon-vector-jni/src/main/java/org/apache/paimon/index/vector/VectorIndexReader.java
+++ /dev/null
@@ -1,196 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.paimon.index.vector;
-
-public final class VectorIndexReader implements AutoCloseable {
-
- private final Object nativeHandleLock = new Object();
- private long nativePtr;
- private Thread nativeHandleOwner;
- private VectorIndexMetadata metadata;
-
- public VectorIndexReader(VectorIndexInput input) {
- if (input == null) {
- throw new NullPointerException("input");
- }
- this.nativePtr = VectorIndexNative.openReader(input);
- }
-
- private VectorIndexReader(long nativePtr) {
- this.nativePtr = nativePtr;
- }
-
- static VectorIndexReader fromNativePointerForTesting(long nativePtr) {
- return new VectorIndexReader(nativePtr);
- }
-
- public VectorIndexMetadata metadata() {
- synchronized (nativeHandleLock) {
- enterNativeHandle();
- try {
- requireOpen();
- if (metadata == null) {
- metadata = VectorIndexNative.metadata(nativePtr);
- }
- return metadata;
- } finally {
- exitNativeHandle();
- }
- }
- }
-
- public String indexType() {
- return metadata().indexType();
- }
-
- public int dimension() {
- return metadata().dimension();
- }
-
- public long totalVectors() {
- return metadata().totalVectors();
- }
-
- public VectorSearchResult search(float[] query, int topK, int nprobe) {
- return search(query, topK, nprobe, 0);
- }
-
- public VectorSearchResult search(float[] query, int topK, int nprobe, int
efSearch) {
- validateQuery(query);
- synchronized (nativeHandleLock) {
- enterNativeHandle();
- try {
- return VectorIndexNative.search(requireOpen(), query, topK,
nprobe, efSearch);
- } finally {
- exitNativeHandle();
- }
- }
- }
-
- public VectorSearchResult search(float[] query, int topK, int nprobe,
byte[] roaringFilter) {
- return search(query, topK, nprobe, 0, roaringFilter);
- }
-
- public VectorSearchResult search(
- float[] query, int topK, int nprobe, int efSearch, byte[]
roaringFilter) {
- validateQuery(query);
- if (roaringFilter == null) {
- throw new NullPointerException("roaringFilter");
- }
- synchronized (nativeHandleLock) {
- enterNativeHandle();
- try {
- return VectorIndexNative.searchWithRoaringFilter(
- requireOpen(), query, topK, nprobe, efSearch,
roaringFilter);
- } finally {
- exitNativeHandle();
- }
- }
- }
-
- public VectorSearchBatchResult searchBatch(
- float[] queries, int queryCount, int topK, int nprobe) {
- return searchBatch(queries, queryCount, topK, nprobe, 0);
- }
-
- public VectorSearchBatchResult searchBatch(
- float[] queries, int queryCount, int topK, int nprobe, int
efSearch) {
- if (queries == null) {
- throw new NullPointerException("queries");
- }
- synchronized (nativeHandleLock) {
- enterNativeHandle();
- try {
- return VectorIndexNative.searchBatch(
- requireOpen(), queries, queryCount, topK, nprobe,
efSearch);
- } finally {
- exitNativeHandle();
- }
- }
- }
-
- public VectorSearchBatchResult searchBatch(
- float[] queries, int queryCount, int topK, int nprobe, byte[]
roaringFilter) {
- return searchBatch(queries, queryCount, topK, nprobe, 0,
roaringFilter);
- }
-
- public VectorSearchBatchResult searchBatch(
- float[] queries,
- int queryCount,
- int topK,
- int nprobe,
- int efSearch,
- byte[] roaringFilter) {
- if (queries == null) {
- throw new NullPointerException("queries");
- }
- if (roaringFilter == null) {
- throw new NullPointerException("roaringFilter");
- }
- synchronized (nativeHandleLock) {
- enterNativeHandle();
- try {
- return VectorIndexNative.searchBatchWithRoaringFilter(
- requireOpen(), queries, queryCount, topK, nprobe,
efSearch, roaringFilter);
- } finally {
- exitNativeHandle();
- }
- }
- }
-
- @Override
- public void close() {
- synchronized (nativeHandleLock) {
- enterNativeHandle();
- try {
- long ptr = nativePtr;
- nativePtr = 0L;
- if (ptr != 0L) {
- VectorIndexNative.freeReader(ptr);
- }
- } finally {
- exitNativeHandle();
- }
- }
- }
-
- private void validateQuery(float[] query) {
- if (query == null) {
- throw new NullPointerException("query");
- }
- }
-
- private long requireOpen() {
- if (nativePtr == 0L) {
- throw new IllegalStateException("VectorIndexReader is closed");
- }
- return nativePtr;
- }
-
- private void enterNativeHandle() {
- Thread current = Thread.currentThread();
- if (nativeHandleOwner == current) {
- throw new IllegalStateException("VectorIndexReader native handle
is already in use");
- }
- nativeHandleOwner = current;
- }
-
- private void exitNativeHandle() {
- nativeHandleOwner = null;
- }
-}
diff --git
a/paimon-vector/paimon-vector-jni/src/main/java/org/apache/paimon/index/vector/VectorIndexWriter.java
b/paimon-vector/paimon-vector-jni/src/main/java/org/apache/paimon/index/vector/VectorIndexWriter.java
deleted file mode 100644
index 0dda2a3eba..0000000000
---
a/paimon-vector/paimon-vector-jni/src/main/java/org/apache/paimon/index/vector/VectorIndexWriter.java
+++ /dev/null
@@ -1,131 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.paimon.index.vector;
-
-import java.util.Map;
-
-public final class VectorIndexWriter implements AutoCloseable {
-
- private final Object nativeHandleLock = new Object();
- private long nativePtr;
- private Thread nativeHandleOwner;
-
- public VectorIndexWriter(Map<String, String> options) {
- String[] keys = new String[options.size()];
- String[] values = new String[options.size()];
- int index = 0;
- for (Map.Entry<String, String> entry : options.entrySet()) {
- keys[index] = entry.getKey();
- values[index] = entry.getValue();
- index++;
- }
- this.nativePtr = VectorIndexNative.createWriter(keys, values);
- }
-
- private VectorIndexWriter(long nativePtr) {
- this.nativePtr = nativePtr;
- }
-
- static VectorIndexWriter fromNativePointerForTesting(long nativePtr) {
- return new VectorIndexWriter(nativePtr);
- }
-
- public int dimension() {
- return VectorIndexNative.writerDimension(requireOpen());
- }
-
- public void train(float[] data, int vectorCount) {
- if (data == null) {
- throw new NullPointerException("data");
- }
- synchronized (nativeHandleLock) {
- enterNativeHandle();
- try {
- VectorIndexNative.train(requireOpen(), data, vectorCount);
- } finally {
- exitNativeHandle();
- }
- }
- }
-
- public void addVectors(long[] ids, float[] data, int vectorCount) {
- if (ids == null) {
- throw new NullPointerException("ids");
- }
- if (data == null) {
- throw new NullPointerException("data");
- }
- synchronized (nativeHandleLock) {
- enterNativeHandle();
- try {
- VectorIndexNative.addVectors(requireOpen(), ids, data,
vectorCount);
- } finally {
- exitNativeHandle();
- }
- }
- }
-
- public void writeIndex(Object output) {
- if (output == null) {
- throw new NullPointerException("output");
- }
- synchronized (nativeHandleLock) {
- enterNativeHandle();
- try {
- VectorIndexNative.writeIndex(requireOpen(), output);
- } finally {
- exitNativeHandle();
- }
- }
- }
-
- @Override
- public void close() {
- synchronized (nativeHandleLock) {
- enterNativeHandle();
- try {
- long ptr = nativePtr;
- nativePtr = 0L;
- if (ptr != 0L) {
- VectorIndexNative.freeWriter(ptr);
- }
- } finally {
- exitNativeHandle();
- }
- }
- }
-
- private long requireOpen() {
- if (nativePtr == 0L) {
- throw new IllegalStateException("VectorIndexWriter is closed");
- }
- return nativePtr;
- }
-
- private void enterNativeHandle() {
- Thread current = Thread.currentThread();
- if (nativeHandleOwner == current) {
- throw new IllegalStateException("VectorIndexWriter native handle
is already in use");
- }
- nativeHandleOwner = current;
- }
-
- private void exitNativeHandle() {
- nativeHandleOwner = null;
- }
-}
diff --git
a/paimon-vector/paimon-vector-jni/src/main/java/org/apache/paimon/index/vector/VectorSearchBatchResult.java
b/paimon-vector/paimon-vector-jni/src/main/java/org/apache/paimon/index/vector/VectorSearchBatchResult.java
deleted file mode 100644
index 12952e932e..0000000000
---
a/paimon-vector/paimon-vector-jni/src/main/java/org/apache/paimon/index/vector/VectorSearchBatchResult.java
+++ /dev/null
@@ -1,100 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.paimon.index.vector;
-
-import java.util.Arrays;
-
-public final class VectorSearchBatchResult {
-
- private final long[] ids;
- private final float[] distances;
- private final int queryCount;
- private final int topK;
-
- public VectorSearchBatchResult(long[] ids, float[] distances, int
queryCount, int topK) {
- if (ids == null) {
- throw new NullPointerException("ids");
- }
- if (distances == null) {
- throw new NullPointerException("distances");
- }
- if (queryCount < 0) {
- throw new IllegalArgumentException("queryCount must be >= 0");
- }
- if (topK < 0) {
- throw new IllegalArgumentException("topK must be >= 0");
- }
- int expectedLength = checkedResultLength(queryCount, topK);
- if (ids.length != expectedLength) {
- throw new IllegalArgumentException(
- "ids length " + ids.length + " != queryCount * topK " +
expectedLength);
- }
- if (distances.length != expectedLength) {
- throw new IllegalArgumentException(
- "distances length "
- + distances.length
- + " != queryCount * topK "
- + expectedLength);
- }
- this.ids = ids.clone();
- this.distances = distances.clone();
- this.queryCount = queryCount;
- this.topK = topK;
- }
-
- public int queryCount() {
- return queryCount;
- }
-
- public int topK() {
- return topK;
- }
-
- public long[] ids() {
- return ids.clone();
- }
-
- public float[] distances() {
- return distances.clone();
- }
-
- public long[] idsForQuery(int queryIndex) {
- checkQueryIndex(queryIndex);
- return Arrays.copyOfRange(ids, queryIndex * topK, (queryIndex + 1) *
topK);
- }
-
- public float[] distancesForQuery(int queryIndex) {
- checkQueryIndex(queryIndex);
- return Arrays.copyOfRange(distances, queryIndex * topK, (queryIndex +
1) * topK);
- }
-
- private void checkQueryIndex(int queryIndex) {
- if (queryIndex < 0 || queryIndex >= queryCount) {
- throw new IndexOutOfBoundsException(
- "queryIndex " + queryIndex + " out of range [0, " +
queryCount + ')');
- }
- }
-
- private static int checkedResultLength(int queryCount, int topK) {
- long length = (long) queryCount * (long) topK;
- if (length > Integer.MAX_VALUE) {
- throw new IllegalArgumentException("queryCount * topK overflows
int");
- }
- return (int) length;
- }
-}
diff --git
a/paimon-vector/paimon-vector-jni/src/main/java/org/apache/paimon/index/vector/VectorSearchResult.java
b/paimon-vector/paimon-vector-jni/src/main/java/org/apache/paimon/index/vector/VectorSearchResult.java
deleted file mode 100644
index 870aa49ce1..0000000000
---
a/paimon-vector/paimon-vector-jni/src/main/java/org/apache/paimon/index/vector/VectorSearchResult.java
+++ /dev/null
@@ -1,62 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.paimon.index.vector;
-
-import java.util.Arrays;
-
-public final class VectorSearchResult {
-
- private final long[] ids;
- private final float[] distances;
-
- public VectorSearchResult(long[] ids, float[] distances) {
- if (ids == null) {
- throw new NullPointerException("ids");
- }
- if (distances == null) {
- throw new NullPointerException("distances");
- }
- if (ids.length != distances.length) {
- throw new IllegalArgumentException(
- "ids length " + ids.length + " != distances length " +
distances.length);
- }
- this.ids = ids.clone();
- this.distances = distances.clone();
- }
-
- public int size() {
- return ids.length;
- }
-
- public long[] ids() {
- return ids.clone();
- }
-
- public float[] distances() {
- return distances.clone();
- }
-
- @Override
- public String toString() {
- return "VectorSearchResult{ids="
- + Arrays.toString(ids)
- + ", distances="
- + Arrays.toString(distances)
- + '}';
- }
-}
diff --git a/paimon-vector/pom.xml b/paimon-vector/pom.xml
index 5cbb01a35a..9fd20e915d 100644
--- a/paimon-vector/pom.xml
+++ b/paimon-vector/pom.xml
@@ -30,10 +30,175 @@ under the License.
<artifactId>paimon-vector</artifactId>
<name>Paimon : Vector Index</name>
- <packaging>pom</packaging>
- <modules>
- <module>paimon-vector-jni</module>
- <module>paimon-vector-index</module>
- </modules>
+ <properties>
+
<paimon-vector-index-java.version>0.1.0</paimon-vector-index-java.version>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.paimon</groupId>
+ <artifactId>paimon-vector-index-java</artifactId>
+ <version>${paimon-vector-index-java.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.paimon</groupId>
+ <artifactId>paimon-common</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.paimon</groupId>
+ <artifactId>paimon-shade-jackson-2</artifactId>
+
<version>${paimon.shade.jackson.version}-${paimon.shade.version}</version>
+ </dependency>
+
+ <!-- test dependencies -->
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter</artifactId>
+ <version>${junit5.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.paimon</groupId>
+ <artifactId>paimon-core</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.paimon</groupId>
+ <artifactId>paimon-core</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.paimon</groupId>
+ <artifactId>paimon-common</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.paimon</groupId>
+ <artifactId>paimon-format</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.paimon</groupId>
+ <artifactId>paimon-test-utils</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs-client</artifactId>
+ <version>${hadoop.version}</version>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.avro</groupId>
+ <artifactId>avro</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>${hadoop.version}</version>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.avro</groupId>
+ <artifactId>avro</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>jdk.tools</groupId>
+ <artifactId>jdk.tools</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ <version>${hadoop.version}</version>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.avro</groupId>
+ <artifactId>avro</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>ch.qos.reload4j</groupId>
+ <artifactId>reload4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-reload4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>jdk.tools</groupId>
+ <artifactId>jdk.tools</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <forkCount>1</forkCount>
+ <redirectTestOutputToFile>true</redirectTestOutputToFile>
+ <parallel>none</parallel>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
</project>
diff --git
a/paimon-vector/paimon-vector-index/src/main/java/org/apache/paimon/vector/index/IvfFlatVectorGlobalIndexerFactory.java
b/paimon-vector/src/main/java/org/apache/paimon/vector/index/IvfFlatVectorGlobalIndexerFactory.java
similarity index 92%
rename from
paimon-vector/paimon-vector-index/src/main/java/org/apache/paimon/vector/index/IvfFlatVectorGlobalIndexerFactory.java
rename to
paimon-vector/src/main/java/org/apache/paimon/vector/index/IvfFlatVectorGlobalIndexerFactory.java
index 572c7cf4ed..12da3c121d 100644
---
a/paimon-vector/paimon-vector-index/src/main/java/org/apache/paimon/vector/index/IvfFlatVectorGlobalIndexerFactory.java
+++
b/paimon-vector/src/main/java/org/apache/paimon/vector/index/IvfFlatVectorGlobalIndexerFactory.java
@@ -19,7 +19,7 @@
package org.apache.paimon.vector.index;
/** Factory for the {@code ivf-flat} vector index identifier. */
-public class IvfFlatVectorGlobalIndexerFactory extends
VectorGlobalIndexerFactory {
+public class IvfFlatVectorGlobalIndexerFactory extends
NativeVectorGlobalIndexerFactory {
public static final String IDENTIFIER = "ivf-flat";
diff --git
a/paimon-vector/paimon-vector-index/src/main/java/org/apache/paimon/vector/index/IvfHnswFlatVectorGlobalIndexerFactory.java
b/paimon-vector/src/main/java/org/apache/paimon/vector/index/IvfHnswFlatVectorGlobalIndexerFactory.java
similarity index 91%
rename from
paimon-vector/paimon-vector-index/src/main/java/org/apache/paimon/vector/index/IvfHnswFlatVectorGlobalIndexerFactory.java
rename to
paimon-vector/src/main/java/org/apache/paimon/vector/index/IvfHnswFlatVectorGlobalIndexerFactory.java
index 159e7af6f1..764920d5d7 100644
---
a/paimon-vector/paimon-vector-index/src/main/java/org/apache/paimon/vector/index/IvfHnswFlatVectorGlobalIndexerFactory.java
+++
b/paimon-vector/src/main/java/org/apache/paimon/vector/index/IvfHnswFlatVectorGlobalIndexerFactory.java
@@ -19,7 +19,7 @@
package org.apache.paimon.vector.index;
/** Factory for the {@code ivf-hnsw-flat} vector index identifier. */
-public class IvfHnswFlatVectorGlobalIndexerFactory extends
VectorGlobalIndexerFactory {
+public class IvfHnswFlatVectorGlobalIndexerFactory extends
NativeVectorGlobalIndexerFactory {
public static final String IDENTIFIER = "ivf-hnsw-flat";
diff --git
a/paimon-vector/paimon-vector-index/src/main/java/org/apache/paimon/vector/index/IvfHnswSqVectorGlobalIndexerFactory.java
b/paimon-vector/src/main/java/org/apache/paimon/vector/index/IvfHnswSqVectorGlobalIndexerFactory.java
similarity index 92%
rename from
paimon-vector/paimon-vector-index/src/main/java/org/apache/paimon/vector/index/IvfHnswSqVectorGlobalIndexerFactory.java
rename to
paimon-vector/src/main/java/org/apache/paimon/vector/index/IvfHnswSqVectorGlobalIndexerFactory.java
index 51c72cd8f3..3f09984db5 100644
---
a/paimon-vector/paimon-vector-index/src/main/java/org/apache/paimon/vector/index/IvfHnswSqVectorGlobalIndexerFactory.java
+++
b/paimon-vector/src/main/java/org/apache/paimon/vector/index/IvfHnswSqVectorGlobalIndexerFactory.java
@@ -19,7 +19,7 @@
package org.apache.paimon.vector.index;
/** Factory for the {@code ivf-hnsw-sq} vector index identifier. */
-public class IvfHnswSqVectorGlobalIndexerFactory extends
VectorGlobalIndexerFactory {
+public class IvfHnswSqVectorGlobalIndexerFactory extends
NativeVectorGlobalIndexerFactory {
public static final String IDENTIFIER = "ivf-hnsw-sq";
diff --git
a/paimon-vector/paimon-vector-index/src/main/java/org/apache/paimon/vector/index/IvfPqAlgorithmVectorGlobalIndexerFactory.java
b/paimon-vector/src/main/java/org/apache/paimon/vector/index/IvfPqAlgorithmVectorGlobalIndexerFactory.java
similarity index 91%
rename from
paimon-vector/paimon-vector-index/src/main/java/org/apache/paimon/vector/index/IvfPqAlgorithmVectorGlobalIndexerFactory.java
rename to
paimon-vector/src/main/java/org/apache/paimon/vector/index/IvfPqAlgorithmVectorGlobalIndexerFactory.java
index f3932de46e..f00adfcbfc 100644
---
a/paimon-vector/paimon-vector-index/src/main/java/org/apache/paimon/vector/index/IvfPqAlgorithmVectorGlobalIndexerFactory.java
+++
b/paimon-vector/src/main/java/org/apache/paimon/vector/index/IvfPqAlgorithmVectorGlobalIndexerFactory.java
@@ -19,7 +19,7 @@
package org.apache.paimon.vector.index;
/** Factory for the {@code ivf-pq} vector index identifier. */
-public class IvfPqAlgorithmVectorGlobalIndexerFactory extends
VectorGlobalIndexerFactory {
+public class IvfPqAlgorithmVectorGlobalIndexerFactory extends
NativeVectorGlobalIndexerFactory {
public static final String IDENTIFIER = "ivf-pq";
diff --git
a/paimon-vector/paimon-vector-index/src/main/java/org/apache/paimon/vector/index/VectorGlobalIndexReader.java
b/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexReader.java
similarity index 97%
rename from
paimon-vector/paimon-vector-index/src/main/java/org/apache/paimon/vector/index/VectorGlobalIndexReader.java
rename to
paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexReader.java
index cde8d2a83d..1f0b46fca9 100644
---
a/paimon-vector/paimon-vector-index/src/main/java/org/apache/paimon/vector/index/VectorGlobalIndexReader.java
+++
b/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexReader.java
@@ -55,12 +55,12 @@ import java.util.concurrent.ExecutorService;
import static org.apache.paimon.utils.Preconditions.checkArgument;
/**
- * Vector global index reader using paimon-vector-index.
+ * Vector global index reader using paimon-vector-index-java.
*
* <p>Each shard has exactly one vector index file. The reader lazily opens
the index and performs
* vector similarity search.
*/
-public class VectorGlobalIndexReader implements GlobalIndexReader {
+public class NativeVectorGlobalIndexReader implements GlobalIndexReader {
private static final String NPROBE_PARAMETER = "ivf.nprobe";
private static final String EF_SEARCH_PARAMETER = "hnsw.ef_search";
@@ -78,7 +78,7 @@ public class VectorGlobalIndexReader implements
GlobalIndexReader {
private volatile VectorIndexReader vectorReader;
private SeekableInputStream openStream;
- public VectorGlobalIndexReader(
+ public NativeVectorGlobalIndexReader(
GlobalIndexFileReader fileReader,
List<GlobalIndexIOMeta> ioMetas,
DataType fieldType,
@@ -198,7 +198,7 @@ public class VectorGlobalIndexReader implements
GlobalIndexReader {
return buildScoredResult(result.ids(), result.distances(),
metric).orElse(null);
}
- private static Optional<ScoredGlobalIndexResult> buildScoredResult(
+ static Optional<ScoredGlobalIndexResult> buildScoredResult(
long[] ids, float[] distances, String metric) {
if (ids.length == 0) {
return Optional.empty();
@@ -274,7 +274,7 @@ public class VectorGlobalIndexReader implements
GlobalIndexReader {
} else if ("cosine".equals(metric)) {
return 1.0f - distance;
} else if ("inner_product".equals(metric)) {
- return distance;
+ return -distance;
}
throw new IllegalArgumentException("Unknown metric: " + metric);
}
@@ -331,9 +331,11 @@ public class VectorGlobalIndexReader implements
GlobalIndexReader {
if (vectorReader == null) {
SeekableInputStream in = fileReader.getInputStream(ioMeta);
try {
- vectorReader =
+ NativeVectorIndexLoader.loadJni();
+ VectorIndexReader reader =
new VectorIndexReader(new
SeekableStreamVectorIndexInput(in));
- nativeMeta = vectorReader.metadata();
+ nativeMeta = reader.metadata();
+ vectorReader = reader;
openStream = in;
} catch (Exception e) {
IOUtils.closeQuietly(in);
diff --git
a/paimon-vector/paimon-vector-index/src/main/java/org/apache/paimon/vector/index/VectorGlobalIndexWriter.java
b/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexWriter.java
similarity index 97%
rename from
paimon-vector/paimon-vector-index/src/main/java/org/apache/paimon/vector/index/VectorGlobalIndexWriter.java
rename to
paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexWriter.java
index 0554bb7862..89aa004bee 100644
---
a/paimon-vector/paimon-vector-index/src/main/java/org/apache/paimon/vector/index/VectorGlobalIndexWriter.java
+++
b/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexWriter.java
@@ -46,7 +46,7 @@ import java.util.List;
import java.util.Map;
/**
- * Vector global index writer using paimon-vector-index.
+ * Vector global index writer using paimon-vector-index-java.
*
* <p>Vectors are spilled to a temporary file on disk as they arrive via
{@link #write(Object,
* long)}, keeping Java heap usage constant (~8 MB buffer). During index
build, vectors are read
@@ -54,11 +54,11 @@ import java.util.Map;
*
* <p><b>Thread safety:</b> This class is <b>not</b> thread-safe.
*/
-public class VectorGlobalIndexWriter implements GlobalIndexSingleColumnWriter,
Closeable {
+public class NativeVectorGlobalIndexWriter implements
GlobalIndexSingleColumnWriter, Closeable {
private static final String FILE_NAME_PREFIX = "vector";
- private static final Logger LOG =
LoggerFactory.getLogger(VectorGlobalIndexWriter.class);
+ private static final Logger LOG =
LoggerFactory.getLogger(NativeVectorGlobalIndexWriter.class);
private static final int IO_BUFFER_SIZE = 8 * 1024 * 1024;
private static final int ADD_BATCH_SIZE = 10000;
@@ -79,7 +79,7 @@ public class VectorGlobalIndexWriter implements
GlobalIndexSingleColumnWriter, C
private long rowCount;
- public VectorGlobalIndexWriter(
+ public NativeVectorGlobalIndexWriter(
GlobalIndexFileWriter fileWriter,
DataType fieldType,
Map<String, String> options,
@@ -95,7 +95,7 @@ public class VectorGlobalIndexWriter implements
GlobalIndexSingleColumnWriter, C
this.vectorBuf = new float[dim];
try {
- this.tempVectorFile =
File.createTempFile("paimon-vector-index-vectors-", ".bin");
+ this.tempVectorFile =
File.createTempFile("paimon-vector-vectors-", ".bin");
this.tempVectorFile.deleteOnExit();
@SuppressWarnings("resource")
RandomAccessFile raf = new RandomAccessFile(tempVectorFile, "rw");
@@ -223,6 +223,7 @@ public class VectorGlobalIndexWriter implements
GlobalIndexSingleColumnWriter, C
LOG.info("{} vector index build started: {} vectors, dim={}",
identifier, count, dim);
long buildStart = System.currentTimeMillis();
+ NativeVectorIndexLoader.loadJni();
try (VectorIndexWriter writer = new VectorIndexWriter(nativeOptions)) {
// Phase 1: Train
diff --git
a/paimon-vector/paimon-vector-index/src/main/java/org/apache/paimon/vector/index/VectorGlobalIndexer.java
b/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexer.java
similarity index 81%
rename from
paimon-vector/paimon-vector-index/src/main/java/org/apache/paimon/vector/index/VectorGlobalIndexer.java
rename to
paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexer.java
index a3f3bf51fb..7095b5bd2e 100644
---
a/paimon-vector/paimon-vector-index/src/main/java/org/apache/paimon/vector/index/VectorGlobalIndexer.java
+++
b/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexer.java
@@ -31,14 +31,15 @@ import java.util.Map;
import java.util.Objects;
import java.util.concurrent.ExecutorService;
-/** Vector global indexer backed by paimon-vector-index. */
-public class VectorGlobalIndexer implements GlobalIndexer {
+/** Native vector global indexer backed by paimon-vector-index-java. */
+public class NativeVectorGlobalIndexer implements GlobalIndexer {
private final DataType fieldType;
private final Map<String, String> options;
private final String identifier;
- public VectorGlobalIndexer(DataType fieldType, Map<String, String>
options, String identifier) {
+ public NativeVectorGlobalIndexer(
+ DataType fieldType, Map<String, String> options, String
identifier) {
this.fieldType = fieldType;
this.options = Objects.requireNonNull(options, "options must not be
null");
this.identifier = Objects.requireNonNull(identifier, "identifier must
not be null");
@@ -46,7 +47,7 @@ public class VectorGlobalIndexer implements GlobalIndexer {
@Override
public GlobalIndexWriter createWriter(GlobalIndexFileWriter fileWriter) {
- return new VectorGlobalIndexWriter(fileWriter, fieldType, options,
identifier);
+ return new NativeVectorGlobalIndexWriter(fileWriter, fieldType,
options, identifier);
}
@Override
@@ -54,6 +55,6 @@ public class VectorGlobalIndexer implements GlobalIndexer {
GlobalIndexFileReader fileReader,
List<GlobalIndexIOMeta> files,
ExecutorService executor) {
- return new VectorGlobalIndexReader(fileReader, files, fieldType,
executor);
+ return new NativeVectorGlobalIndexReader(fileReader, files, fieldType,
executor);
}
}
diff --git
a/paimon-vector/paimon-vector-index/src/main/java/org/apache/paimon/vector/index/VectorGlobalIndexerFactory.java
b/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexerFactory.java
similarity index 96%
rename from
paimon-vector/paimon-vector-index/src/main/java/org/apache/paimon/vector/index/VectorGlobalIndexerFactory.java
rename to
paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexerFactory.java
index 9367723114..8e4daa030f 100644
---
a/paimon-vector/paimon-vector-index/src/main/java/org/apache/paimon/vector/index/VectorGlobalIndexerFactory.java
+++
b/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexerFactory.java
@@ -28,15 +28,15 @@ import org.apache.paimon.types.VectorType;
import java.util.LinkedHashMap;
import java.util.Map;
-/** Factory for creating vector indexes backed by paimon-vector-index. */
-public abstract class VectorGlobalIndexerFactory implements
GlobalIndexerFactory {
+/** Factory for creating vector indexes backed by paimon-vector-index-java. */
+public abstract class NativeVectorGlobalIndexerFactory implements
GlobalIndexerFactory {
private static final int DEFAULT_DIMENSION = 128;
@Override
public GlobalIndexer create(DataField field, Options options) {
String identifier = identifier();
- return new VectorGlobalIndexer(
+ return new NativeVectorGlobalIndexer(
field.type(),
nativeOptions(field.type(), options, identifier, field.name()),
identifier);
diff --git
a/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeVectorIndexLoader.java
b/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeVectorIndexLoader.java
new file mode 100644
index 0000000000..3c55fda25f
--- /dev/null
+++
b/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeVectorIndexLoader.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.vector.index;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Locale;
+
+/** Loads native libraries packaged by paimon-vector-index-java. */
+final class NativeVectorIndexLoader {
+
+ private static volatile boolean loaded;
+
+ private NativeVectorIndexLoader() {}
+
+ static void loadJni() {
+ if (loaded) {
+ return;
+ }
+ synchronized (NativeVectorIndexLoader.class) {
+ if (loaded) {
+ return;
+ }
+ loadFromResource();
+ loaded = true;
+ }
+ }
+
+ private static void loadFromResource() {
+ String libraryPath = nativeLibraryPath();
+ try (InputStream input =
NativeVectorIndexLoader.class.getResourceAsStream(libraryPath)) {
+ if (input == null) {
+ throw new UnsupportedOperationException(
+ "Vector index native library resource not found: " +
libraryPath);
+ }
+
+ File temp = File.createTempFile("paimon-vector-", librarySuffix());
+ temp.deleteOnExit();
+ try (FileOutputStream output = new FileOutputStream(temp)) {
+ byte[] buffer = new byte[8192];
+ int len;
+ while ((len = input.read(buffer)) >= 0) {
+ output.write(buffer, 0, len);
+ }
+ }
+ System.load(temp.getAbsolutePath());
+ } catch (IOException e) {
+ throw new RuntimeException(
+ "Failed to load vector index native library: " +
libraryPath, e);
+ }
+ }
+
+ private static String nativeLibraryPath() {
+ return "/native/" + os() + "/" + arch() + "/" + libraryName();
+ }
+
+ private static String os() {
+ String os = System.getProperty("os.name").toLowerCase(Locale.ROOT);
+ if (os.contains("mac")) {
+ return "macos";
+ } else if (os.contains("linux")) {
+ return "linux";
+ } else if (os.contains("windows")) {
+ return "windows";
+ }
+ throw new UnsupportedOperationException("Unsupported vector index
native OS: " + os);
+ }
+
+ private static String arch() {
+ String arch = System.getProperty("os.arch").toLowerCase(Locale.ROOT);
+ if ("amd64".equals(arch) || "x86_64".equals(arch)) {
+ return "x86_64";
+ } else if ("aarch64".equals(arch) || "arm64".equals(arch)) {
+ return "aarch64";
+ }
+ throw new UnsupportedOperationException("Unsupported vector index
native arch: " + arch);
+ }
+
+ private static String libraryName() {
+ if ("windows".equals(os())) {
+ return "paimon_vindex_jni.dll";
+ }
+ return "libpaimon_vindex_jni" + librarySuffix();
+ }
+
+ private static String librarySuffix() {
+ if ("macos".equals(os())) {
+ return ".dylib";
+ } else if ("windows".equals(os())) {
+ return ".dll";
+ }
+ return ".so";
+ }
+}
diff --git
a/paimon-vector/paimon-vector-index/src/main/java/org/apache/paimon/vector/index/VectorIndexMeta.java
b/paimon-vector/src/main/java/org/apache/paimon/vector/index/VectorIndexMeta.java
similarity index 100%
rename from
paimon-vector/paimon-vector-index/src/main/java/org/apache/paimon/vector/index/VectorIndexMeta.java
rename to
paimon-vector/src/main/java/org/apache/paimon/vector/index/VectorIndexMeta.java
diff --git
a/paimon-vector/paimon-vector-index/src/main/resources/META-INF/services/org.apache.paimon.globalindex.GlobalIndexerFactory
b/paimon-vector/src/main/resources/META-INF/services/org.apache.paimon.globalindex.GlobalIndexerFactory
similarity index 100%
rename from
paimon-vector/paimon-vector-index/src/main/resources/META-INF/services/org.apache.paimon.globalindex.GlobalIndexerFactory
rename to
paimon-vector/src/main/resources/META-INF/services/org.apache.paimon.globalindex.GlobalIndexerFactory
diff --git a/paimon-vector/src/test/java/org/apache/paimon/JavaPyE2ETest.java
b/paimon-vector/src/test/java/org/apache/paimon/JavaPyE2ETest.java
new file mode 100644
index 0000000000..033245f9eb
--- /dev/null
+++ b/paimon-vector/src/test/java/org/apache/paimon/JavaPyE2ETest.java
@@ -0,0 +1,200 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon;
+
+import org.apache.paimon.data.BinaryRow;
+import org.apache.paimon.data.GenericArray;
+import org.apache.paimon.data.GenericRow;
+import org.apache.paimon.fs.FileIOFinder;
+import org.apache.paimon.fs.Path;
+import org.apache.paimon.fs.local.LocalFileIO;
+import org.apache.paimon.globalindex.GlobalIndexBuilderUtils;
+import org.apache.paimon.globalindex.GlobalIndexSingleColumnWriter;
+import org.apache.paimon.globalindex.ResultEntry;
+import org.apache.paimon.index.IndexFileMeta;
+import org.apache.paimon.io.CompactIncrement;
+import org.apache.paimon.io.DataIncrement;
+import org.apache.paimon.options.Options;
+import org.apache.paimon.schema.Schema;
+import org.apache.paimon.schema.SchemaManager;
+import org.apache.paimon.schema.SchemaUtils;
+import org.apache.paimon.schema.TableSchema;
+import org.apache.paimon.table.AppendOnlyFileStoreTable;
+import org.apache.paimon.table.CatalogEnvironment;
+import org.apache.paimon.table.sink.BatchTableCommit;
+import org.apache.paimon.table.sink.BatchTableWrite;
+import org.apache.paimon.table.sink.BatchWriteBuilder;
+import org.apache.paimon.table.sink.CommitMessage;
+import org.apache.paimon.table.sink.CommitMessageImpl;
+import org.apache.paimon.types.ArrayType;
+import org.apache.paimon.types.DataField;
+import org.apache.paimon.types.DataType;
+import org.apache.paimon.types.DataTypes;
+import org.apache.paimon.types.FloatType;
+import org.apache.paimon.types.RowType;
+import org.apache.paimon.utils.Range;
+import org.apache.paimon.vector.index.IvfFlatVectorGlobalIndexerFactory;
+
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.condition.EnabledIfSystemProperty;
+
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.Collections;
+import java.util.List;
+
+import static org.apache.paimon.CoreOptions.DATA_EVOLUTION_ENABLED;
+import static org.apache.paimon.CoreOptions.GLOBAL_INDEX_ENABLED;
+import static org.apache.paimon.CoreOptions.PATH;
+import static org.apache.paimon.CoreOptions.ROW_TRACKING_ENABLED;
+import static org.assertj.core.api.Assertions.assertThat;
+
+/**
+ * Mixed language E2E test for Java paimon-vindex building and Python reading.
+ *
+ * <p>Java writes data and builds an ivf-flat vector index, then Python reads
and searches it.
+ */
+public class JavaPyE2ETest {
+
+ java.nio.file.Path tempDir =
Paths.get("../paimon-python/pypaimon/tests/e2e").toAbsolutePath();
+
+ protected Path warehouse;
+
+ @BeforeEach
+ public void before() throws Exception {
+ if (!Files.exists(tempDir.resolve("warehouse"))) {
+ Files.createDirectories(tempDir.resolve("warehouse"));
+ }
+ warehouse = new Path(tempDir.resolve("warehouse").toUri());
+ }
+
+ @Test
+ @EnabledIfSystemProperty(named = "run.e2e.tests", matches = "true")
+ public void testVindexVectorIndexWrite() throws Exception {
+ String tableName = "test_vindex_vector";
+ Path tablePath = new Path(warehouse.toString() + "/default.db/" +
tableName);
+ LocalFileIO fileIO = LocalFileIO.create();
+ if (fileIO.exists(tablePath)) {
+ fileIO.delete(tablePath, true);
+ }
+
+ int dimension = 4;
+
+ RowType rowType =
+ RowType.of(
+ new DataType[] {DataTypes.INT(), new ArrayType(new
FloatType())},
+ new String[] {"id", "embedding"});
+
+ Options options = new Options();
+ options.set(PATH, tablePath.toString());
+ options.set(ROW_TRACKING_ENABLED, true);
+ options.set(DATA_EVOLUTION_ENABLED, true);
+ options.set(GLOBAL_INDEX_ENABLED, true);
+ options.setString(
+ IvfFlatVectorGlobalIndexerFactory.IDENTIFIER + ".dimension",
+ String.valueOf(dimension));
+ options.setString(IvfFlatVectorGlobalIndexerFactory.IDENTIFIER +
".metric", "l2");
+ options.setString(IvfFlatVectorGlobalIndexerFactory.IDENTIFIER +
".nlist", "2");
+
+ TableSchema tableSchema =
+ SchemaUtils.forceCommit(
+ new SchemaManager(fileIO, tablePath),
+ new Schema(
+ rowType.getFields(),
+ Collections.emptyList(),
+ Collections.emptyList(),
+ options.toMap(),
+ ""));
+
+ AppendOnlyFileStoreTable table =
+ new AppendOnlyFileStoreTable(
+ FileIOFinder.find(tablePath),
+ tablePath,
+ tableSchema,
+ CatalogEnvironment.empty());
+
+ float[][] vectors =
+ new float[][] {
+ new float[] {1.0f, 0.0f, 0.0f, 0.0f},
+ new float[] {0.9f, 0.1f, 0.0f, 0.0f},
+ new float[] {0.0f, 1.0f, 0.0f, 0.0f},
+ new float[] {0.0f, 0.0f, 1.0f, 0.0f},
+ new float[] {0.0f, 0.0f, 0.0f, 1.0f},
+ new float[] {0.95f, 0.05f, 0.0f, 0.0f}
+ };
+
+ BatchWriteBuilder writeBuilder = table.newBatchWriteBuilder();
+ try (BatchTableWrite write = writeBuilder.newWrite();
+ BatchTableCommit commit = writeBuilder.newCommit()) {
+ for (int i = 0; i < vectors.length; i++) {
+ write.write(GenericRow.of(i, new GenericArray(vectors[i])));
+ }
+ commit.commit(write.prepareCommit());
+ }
+
+ DataField embeddingField = table.rowType().getField("embedding");
+ Options indexOptions = table.coreOptions().toConfiguration();
+
+ GlobalIndexSingleColumnWriter writer =
+ (GlobalIndexSingleColumnWriter)
+ GlobalIndexBuilderUtils.createIndexWriter(
+ table,
+ IvfFlatVectorGlobalIndexerFactory.IDENTIFIER,
+ embeddingField,
+ indexOptions);
+
+ for (int i = 0; i < vectors.length; i++) {
+ writer.write(vectors[i], i);
+ }
+
+ List<ResultEntry> entries = writer.finish();
+ assertThat(entries).hasSize(1);
+ assertThat(entries.get(0).rowCount()).isEqualTo(vectors.length);
+
+ Range rowRange = new Range(0, vectors.length - 1);
+ List<IndexFileMeta> indexFiles =
+ GlobalIndexBuilderUtils.toIndexFileMetas(
+ table.fileIO(),
+ table.store().pathFactory().globalIndexFileFactory(),
+ table.coreOptions(),
+ rowRange,
+ embeddingField.id(),
+ IvfFlatVectorGlobalIndexerFactory.IDENTIFIER,
+ entries);
+
+ DataIncrement dataIncrement = DataIncrement.indexIncrement(indexFiles);
+ CommitMessage message =
+ new CommitMessageImpl(
+ BinaryRow.EMPTY_ROW,
+ 0,
+ null,
+ dataIncrement,
+ CompactIncrement.emptyIncrement());
+ try (BatchTableCommit commit = writeBuilder.newCommit()) {
+ commit.commit(Collections.singletonList(message));
+ }
+
+ List<org.apache.paimon.manifest.IndexManifestEntry> indexEntries =
+
table.indexManifestFileReader().read(table.latestSnapshot().get().indexManifest());
+ assertThat(indexEntries).hasSize(1);
+ assertThat(indexEntries.get(0).indexFile().indexType())
+ .isEqualTo(IvfFlatVectorGlobalIndexerFactory.IDENTIFIER);
+ }
+}
diff --git
a/paimon-vector/paimon-vector-index/src/test/java/org/apache/paimon/vector/index/VectorGlobalIndexTest.java
b/paimon-vector/src/test/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexTest.java
similarity index 81%
rename from
paimon-vector/paimon-vector-index/src/test/java/org/apache/paimon/vector/index/VectorGlobalIndexTest.java
rename to
paimon-vector/src/test/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexTest.java
index f08c072984..72303f5bf0 100644
---
a/paimon-vector/paimon-vector-index/src/test/java/org/apache/paimon/vector/index/VectorGlobalIndexTest.java
+++
b/paimon-vector/src/test/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexTest.java
@@ -27,7 +27,6 @@ import org.apache.paimon.globalindex.ResultEntry;
import org.apache.paimon.globalindex.ScoredGlobalIndexResult;
import org.apache.paimon.globalindex.io.GlobalIndexFileReader;
import org.apache.paimon.globalindex.io.GlobalIndexFileWriter;
-import org.apache.paimon.index.vector.NativeLoader;
import org.apache.paimon.options.Options;
import org.apache.paimon.predicate.BatchVectorSearch;
import org.apache.paimon.predicate.VectorSearch;
@@ -58,8 +57,8 @@ import java.util.concurrent.Executors;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
-/** Tests for {@link VectorGlobalIndexWriter} and {@link
VectorGlobalIndexReader}. */
-public class VectorGlobalIndexTest {
+/** Tests for {@link NativeVectorGlobalIndexWriter} and {@link
NativeVectorGlobalIndexReader}. */
+public class NativeVectorGlobalIndexTest {
@TempDir java.nio.file.Path tempDir;
@@ -73,7 +72,20 @@ public class VectorGlobalIndexTest {
private static boolean isNativeAvailable() {
try {
- NativeLoader.loadJni();
+ NativeVectorIndexLoader.loadJni();
+ Options options = new Options();
+ options.setInteger("ivf-flat.dimension", 2);
+ options.setString("ivf-flat.metric", "l2");
+ options.setInteger("ivf-flat.nlist", 1);
+ try (org.apache.paimon.index.vector.VectorIndexWriter ignored =
+ new org.apache.paimon.index.vector.VectorIndexWriter(
+ NativeVectorGlobalIndexerFactory.nativeOptions(
+ new ArrayType(new FloatType()),
+ options,
+
IvfFlatVectorGlobalIndexerFactory.IDENTIFIER,
+ "vec"))) {
+ // Closed immediately; constructing the writer is enough to
validate JNI loading.
+ }
return true;
} catch (Throwable t) {
return false;
@@ -104,7 +116,7 @@ public class VectorGlobalIndexTest {
public void testDimensionMismatch() {
Options options = createDefaultOptions(64);
GlobalIndexFileWriter fileWriter = createFileWriter(indexPath);
- VectorGlobalIndexWriter writer = createIvfPqWriter(fileWriter,
vectorType, options);
+ NativeVectorGlobalIndexWriter writer = createIvfPqWriter(fileWriter,
vectorType, options);
float[] wrongDimVector = new float[32];
assertThatThrownBy(() -> writer.write(wrongDimVector, 0))
@@ -129,7 +141,7 @@ public class VectorGlobalIndexTest {
Options options = createDefaultOptions(2);
options.setInteger("ivf-pq.pq.m", 1);
GlobalIndexFileWriter fileWriter = createFileWriter(indexPath);
- VectorGlobalIndexWriter writer = createIvfPqWriter(fileWriter,
vectorType, options);
+ NativeVectorGlobalIndexWriter writer = createIvfPqWriter(fileWriter,
vectorType, options);
assertThatThrownBy(() -> writer.write(new float[] {1.0f, Float.NaN},
0))
.isInstanceOf(IllegalArgumentException.class)
@@ -143,7 +155,7 @@ public class VectorGlobalIndexTest {
Options options = createDefaultOptions(2);
options.setInteger("ivf-pq.pq.m", 1);
GlobalIndexFileWriter fileWriter = createFileWriter(indexPath);
- VectorGlobalIndexWriter writer = createIvfPqWriter(fileWriter,
vectorType, options);
+ NativeVectorGlobalIndexWriter writer = createIvfPqWriter(fileWriter,
vectorType, options);
writer.write(null, 0); // row 0 - null
assertThatThrownBy(() -> writer.write(new float[]
{Float.POSITIVE_INFINITY, 0.0f}, 1))
@@ -158,7 +170,7 @@ public class VectorGlobalIndexTest {
Options options = createDefaultOptions(2);
options.setInteger("ivf-pq.pq.m", 1);
GlobalIndexFileWriter fileWriter = createFileWriter(indexPath);
- VectorGlobalIndexWriter writer = createIvfPqWriter(fileWriter,
vectorType, options);
+ NativeVectorGlobalIndexWriter writer = createIvfPqWriter(fileWriter,
vectorType, options);
writer.write(null, 0);
writer.write(null, 1);
@@ -185,22 +197,42 @@ public class VectorGlobalIndexTest {
parameters.put("hnsw.ef_search", "80");
parameters.put("ignored", "bad");
- assertThat(VectorGlobalIndexReader.nprobe(parameters)).isEqualTo(24);
- assertThat(VectorGlobalIndexReader.efSearch(parameters)).isEqualTo(80);
-
assertThat(VectorGlobalIndexReader.nprobe(Collections.emptyMap())).isEqualTo(16);
-
assertThat(VectorGlobalIndexReader.efSearch(Collections.emptyMap())).isEqualTo(0);
+
assertThat(NativeVectorGlobalIndexReader.nprobe(parameters)).isEqualTo(24);
+
assertThat(NativeVectorGlobalIndexReader.efSearch(parameters)).isEqualTo(80);
+
assertThat(NativeVectorGlobalIndexReader.nprobe(Collections.emptyMap())).isEqualTo(16);
+
assertThat(NativeVectorGlobalIndexReader.efSearch(Collections.emptyMap())).isEqualTo(0);
}
@Test
public void testVectorSearchParameterRangeValidationDelegatedToNative() {
-
assertThat(VectorGlobalIndexReader.nprobe(Collections.singletonMap("ivf.nprobe",
"0")))
+ assertThat(
+ NativeVectorGlobalIndexReader.nprobe(
+ Collections.singletonMap("ivf.nprobe", "0")))
.isEqualTo(0);
assertThat(
- VectorGlobalIndexReader.efSearch(
+ NativeVectorGlobalIndexReader.efSearch(
Collections.singletonMap("hnsw.ef_search",
"-1")))
.isEqualTo(-1);
}
+ @Test
+ public void testInnerProductDistanceConvertedToHigherIsBetterScore() {
+ ScoredGlobalIndexResult result =
+ NativeVectorGlobalIndexReader.buildScoredResult(
+ new long[] {10L, 20L, 30L},
+ new float[] {-1.0f, -0.5f, -0.1f},
+ "inner_product")
+ .get();
+
+ assertThat(result.scoreGetter().score(10L)).isEqualTo(1.0f);
+ assertThat(result.scoreGetter().score(20L)).isEqualTo(0.5f);
+ assertThat(result.scoreGetter().score(30L)).isEqualTo(0.1f);
+
+ ScoredGlobalIndexResult top1 = result.topK(1);
+ assertThat(top1.results().getLongCardinality()).isEqualTo(1);
+ assertThat(top1.results().contains(10L)).isTrue();
+ }
+
// =================== Tests that NEED native library =====================
@Test
@@ -223,7 +255,7 @@ public class VectorGlobalIndexTest {
};
GlobalIndexFileWriter fileWriter = createFileWriter(indexPath);
- VectorGlobalIndexWriter writer = createIvfPqWriter(fileWriter,
vectorType, options);
+ NativeVectorGlobalIndexWriter writer = createIvfPqWriter(fileWriter,
vectorType, options);
for (int i = 0; i < vectors.length; i++) {
writer.write(vectors[i], i);
}
@@ -231,8 +263,8 @@ public class VectorGlobalIndexTest {
List<GlobalIndexIOMeta> metas = toIOMetas(results, indexPath);
GlobalIndexFileReader fileReader = createFileReader(indexPath);
- try (VectorGlobalIndexReader reader =
- new VectorGlobalIndexReader(fileReader, metas, vectorType,
executor)) {
+ try (NativeVectorGlobalIndexReader reader =
+ new NativeVectorGlobalIndexReader(fileReader, metas,
vectorType, executor)) {
VectorSearch vectorSearch = new VectorSearch(vectors[0], 3,
fieldName);
ScoredGlobalIndexResult result =
reader.visitVectorSearch(vectorSearch).join().get();
assertThat(result.results().getLongCardinality()).isEqualTo(3);
@@ -262,7 +294,7 @@ public class VectorGlobalIndexTest {
};
GlobalIndexFileWriter fileWriter = createFileWriter(indexPath);
- VectorGlobalIndexWriter writer = createIvfPqWriter(fileWriter,
vectorType, options);
+ NativeVectorGlobalIndexWriter writer = createIvfPqWriter(fileWriter,
vectorType, options);
for (int i = 0; i < vectors.length; i++) {
writer.write(vectors[i], i);
}
@@ -270,8 +302,8 @@ public class VectorGlobalIndexTest {
List<GlobalIndexIOMeta> metas = toIOMetas(results, indexPath);
GlobalIndexFileReader fileReader = createFileReader(indexPath);
- try (VectorGlobalIndexReader reader =
- new VectorGlobalIndexReader(fileReader, metas, vectorType,
executor)) {
+ try (NativeVectorGlobalIndexReader reader =
+ new NativeVectorGlobalIndexReader(fileReader, metas,
vectorType, executor)) {
// Filter to rows {1, 4} only
RoaringNavigableMap64 filter = new RoaringNavigableMap64();
@@ -303,7 +335,7 @@ public class VectorGlobalIndexTest {
};
GlobalIndexFileWriter fileWriter = createFileWriter(indexPath);
- VectorGlobalIndexWriter writer = createIvfPqWriter(fileWriter,
vectorType, options);
+ NativeVectorGlobalIndexWriter writer = createIvfPqWriter(fileWriter,
vectorType, options);
writer.write(vectors[0], 0); // row 0
writer.write(null, 1); // row 1 - null
@@ -318,8 +350,8 @@ public class VectorGlobalIndexTest {
List<GlobalIndexIOMeta> metas = toIOMetas(results, indexPath);
GlobalIndexFileReader fileReader = createFileReader(indexPath);
- try (VectorGlobalIndexReader reader =
- new VectorGlobalIndexReader(fileReader, metas, vectorType,
executor)) {
+ try (NativeVectorGlobalIndexReader reader =
+ new NativeVectorGlobalIndexReader(fileReader, metas,
vectorType, executor)) {
VectorSearch vectorSearch = new VectorSearch(vectors[0], 3,
fieldName);
ScoredGlobalIndexResult result =
reader.visitVectorSearch(vectorSearch).join().get();
assertThat(result.results().getLongCardinality()).isEqualTo(3);
@@ -348,15 +380,16 @@ public class VectorGlobalIndexTest {
new float[] {0.7f, 0.7f}
};
- VectorGlobalIndexer indexer =
- new VectorGlobalIndexer(
+ NativeVectorGlobalIndexer indexer =
+ new NativeVectorGlobalIndexer(
vectorType,
- VectorGlobalIndexerFactory.nativeOptions(
+ NativeVectorGlobalIndexerFactory.nativeOptions(
vectorType, options, IVF_PQ_IDENTIFIER,
fieldName),
IVF_PQ_IDENTIFIER);
GlobalIndexFileWriter fileWriter = createFileWriter(indexPath);
- VectorGlobalIndexWriter writer = (VectorGlobalIndexWriter)
indexer.createWriter(fileWriter);
+ NativeVectorGlobalIndexWriter writer =
+ (NativeVectorGlobalIndexWriter)
indexer.createWriter(fileWriter);
for (int i = 0; i < vectors.length; i++) {
writer.write(vectors[i], i);
}
@@ -364,8 +397,8 @@ public class VectorGlobalIndexTest {
List<GlobalIndexIOMeta> metas = toIOMetas(results, indexPath);
GlobalIndexFileReader fileReader = createFileReader(indexPath);
- try (VectorGlobalIndexReader reader =
- (VectorGlobalIndexReader) indexer.createReader(fileReader,
metas, executor)) {
+ try (NativeVectorGlobalIndexReader reader =
+ (NativeVectorGlobalIndexReader)
indexer.createReader(fileReader, metas, executor)) {
VectorSearch vectorSearch = new VectorSearch(vectors[0], 2,
fieldName);
ScoredGlobalIndexResult result =
reader.visitVectorSearch(vectorSearch).join().get();
assertThat(result.results().getLongCardinality()).isEqualTo(2);
@@ -393,14 +426,14 @@ public class VectorGlobalIndexTest {
};
GlobalIndexFileWriter fileWriter = createFileWriter(indexPath);
- VectorGlobalIndexWriter writer = createIvfPqWriter(fileWriter,
vectorType, options);
+ NativeVectorGlobalIndexWriter writer = createIvfPqWriter(fileWriter,
vectorType, options);
writeVectors(writer, vectors);
List<ResultEntry> results = writer.finish();
List<GlobalIndexIOMeta> metas = toIOMetas(results, indexPath);
GlobalIndexFileReader fileReader = createFileReader(indexPath);
- try (VectorGlobalIndexReader reader =
- new VectorGlobalIndexReader(fileReader, metas, vectorType,
executor)) {
+ try (NativeVectorGlobalIndexReader reader =
+ new NativeVectorGlobalIndexReader(fileReader, metas,
vectorType, executor)) {
float[][] queryVectors =
new float[][] {
new float[] {1.0f, 0.0f},
@@ -445,14 +478,14 @@ public class VectorGlobalIndexTest {
};
GlobalIndexFileWriter fileWriter = createFileWriter(indexPath);
- VectorGlobalIndexWriter writer = createIvfPqWriter(fileWriter,
vectorType, options);
+ NativeVectorGlobalIndexWriter writer = createIvfPqWriter(fileWriter,
vectorType, options);
writeVectors(writer, vectors);
List<ResultEntry> results = writer.finish();
List<GlobalIndexIOMeta> metas = toIOMetas(results, indexPath);
GlobalIndexFileReader fileReader = createFileReader(indexPath);
- try (VectorGlobalIndexReader reader =
- new VectorGlobalIndexReader(fileReader, metas, vectorType,
executor)) {
+ try (NativeVectorGlobalIndexReader reader =
+ new NativeVectorGlobalIndexReader(fileReader, metas,
vectorType, executor)) {
float[][] queryVectors =
new float[][] {new float[] {1.0f, 0.0f}, new float[]
{-1.0f, 0.0f}};
@@ -500,14 +533,14 @@ public class VectorGlobalIndexTest {
};
GlobalIndexFileWriter fileWriter = createFileWriter(indexPath);
- VectorGlobalIndexWriter writer = createIvfPqWriter(fileWriter,
vectorType, options);
+ NativeVectorGlobalIndexWriter writer = createIvfPqWriter(fileWriter,
vectorType, options);
writeVectors(writer, vectors);
List<ResultEntry> results = writer.finish();
List<GlobalIndexIOMeta> metas = toIOMetas(results, indexPath);
GlobalIndexFileReader fileReader = createFileReader(indexPath);
- try (VectorGlobalIndexReader reader =
- new VectorGlobalIndexReader(fileReader, metas, vectorType,
executor)) {
+ try (NativeVectorGlobalIndexReader reader =
+ new NativeVectorGlobalIndexReader(fileReader, metas,
vectorType, executor)) {
float[][] queryVectors =
new float[][] {
new float[] {1.0f, 0.0f},
@@ -541,12 +574,12 @@ public class VectorGlobalIndexTest {
// =================== Helpers =====================
- private VectorGlobalIndexWriter createIvfPqWriter(
+ private NativeVectorGlobalIndexWriter createIvfPqWriter(
GlobalIndexFileWriter fileWriter, DataType fieldType, Options
options) {
- return new VectorGlobalIndexWriter(
+ return new NativeVectorGlobalIndexWriter(
fileWriter,
fieldType,
- VectorGlobalIndexerFactory.nativeOptions(
+ NativeVectorGlobalIndexerFactory.nativeOptions(
fieldType, options, IVF_PQ_IDENTIFIER, fieldName),
IVF_PQ_IDENTIFIER);
}
@@ -558,7 +591,7 @@ public class VectorGlobalIndexTest {
return options;
}
- private void writeVectors(VectorGlobalIndexWriter writer, float[][]
vectors) {
+ private void writeVectors(NativeVectorGlobalIndexWriter writer, float[][]
vectors) {
for (int i = 0; i < vectors.length; i++) {
writer.write(vectors[i], i);
}
diff --git
a/paimon-vector/paimon-vector-index/src/test/java/org/apache/paimon/vector/index/VectorGlobalIndexerFactoryTest.java
b/paimon-vector/src/test/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexerFactoryTest.java
similarity index 92%
rename from
paimon-vector/paimon-vector-index/src/test/java/org/apache/paimon/vector/index/VectorGlobalIndexerFactoryTest.java
rename to
paimon-vector/src/test/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexerFactoryTest.java
index b0f33d7706..92c56b6485 100644
---
a/paimon-vector/paimon-vector-index/src/test/java/org/apache/paimon/vector/index/VectorGlobalIndexerFactoryTest.java
+++
b/paimon-vector/src/test/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexerFactoryTest.java
@@ -32,7 +32,7 @@ import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
/** Tests for vector global indexer factory SPI registration. */
-public class VectorGlobalIndexerFactoryTest {
+public class NativeVectorGlobalIndexerFactoryTest {
@Test
public void testIdentifier() {
@@ -67,7 +67,7 @@ public class VectorGlobalIndexerFactoryTest {
options.setString("ivf-pq.nlist", "256");
Map<String, String> nativeOptions =
- VectorGlobalIndexerFactory.nativeOptions(
+ NativeVectorGlobalIndexerFactory.nativeOptions(
new ArrayType(new FloatType()),
options,
IvfFlatVectorGlobalIndexerFactory.IDENTIFIER,
@@ -90,7 +90,7 @@ public class VectorGlobalIndexerFactoryTest {
options.setString("ivf-flat.dimension", "32");
Map<String, String> nativeOptions =
- VectorGlobalIndexerFactory.nativeOptions(
+ NativeVectorGlobalIndexerFactory.nativeOptions(
new VectorType(8, new FloatType()),
options,
IvfFlatVectorGlobalIndexerFactory.IDENTIFIER,
@@ -106,7 +106,7 @@ public class VectorGlobalIndexerFactoryTest {
assertThatThrownBy(
() ->
- VectorGlobalIndexerFactory.nativeOptions(
+ NativeVectorGlobalIndexerFactory.nativeOptions(
new ArrayType(new FloatType()),
options,
IvfFlatVectorGlobalIndexerFactory.IDENTIFIER,
@@ -124,7 +124,7 @@ public class VectorGlobalIndexerFactoryTest {
options.setString("fields.vec.nlist", "256");
Map<String, String> nativeOptions =
- VectorGlobalIndexerFactory.nativeOptions(
+ NativeVectorGlobalIndexerFactory.nativeOptions(
new ArrayType(new FloatType()),
options,
IvfFlatVectorGlobalIndexerFactory.IDENTIFIER,
@@ -143,7 +143,7 @@ public class VectorGlobalIndexerFactoryTest {
options.setString("fields.vec.dimension", "64");
Map<String, String> nativeOptions =
- VectorGlobalIndexerFactory.nativeOptions(
+ NativeVectorGlobalIndexerFactory.nativeOptions(
new ArrayType(new FloatType()),
options,
IvfFlatVectorGlobalIndexerFactory.IDENTIFIER,
@@ -159,7 +159,7 @@ public class VectorGlobalIndexerFactoryTest {
options.setString("fields.vec.nlist", "256");
Map<String, String> nativeOptions =
- VectorGlobalIndexerFactory.nativeOptions(
+ NativeVectorGlobalIndexerFactory.nativeOptions(
new ArrayType(new FloatType()),
options,
IvfFlatVectorGlobalIndexerFactory.IDENTIFIER,
@@ -175,7 +175,7 @@ public class VectorGlobalIndexerFactoryTest {
options.setString("fields.vec_extra.nlist", "512");
Map<String, String> nativeOptions =
- VectorGlobalIndexerFactory.nativeOptions(
+ NativeVectorGlobalIndexerFactory.nativeOptions(
new ArrayType(new FloatType()),
options,
IvfFlatVectorGlobalIndexerFactory.IDENTIFIER,
@@ -190,7 +190,7 @@ public class VectorGlobalIndexerFactoryTest {
options.setString("fields.vec.distance.metric", "cosine");
Map<String, String> nativeOptions =
- VectorGlobalIndexerFactory.nativeOptions(
+ NativeVectorGlobalIndexerFactory.nativeOptions(
new ArrayType(new FloatType()),
options,
IvfFlatVectorGlobalIndexerFactory.IDENTIFIER,
@@ -207,7 +207,7 @@ public class VectorGlobalIndexerFactoryTest {
options.setString("fields.vec.aggregate-function", "sum");
Map<String, String> nativeOptions =
- VectorGlobalIndexerFactory.nativeOptions(
+ NativeVectorGlobalIndexerFactory.nativeOptions(
new ArrayType(new FloatType()),
options,
IvfFlatVectorGlobalIndexerFactory.IDENTIFIER,
diff --git
a/paimon-vector/paimon-vector-index/src/test/java/org/apache/paimon/vector/index/SeekableStreamVectorIndexInputTest.java
b/paimon-vector/src/test/java/org/apache/paimon/vector/index/SeekableStreamVectorIndexInputTest.java
similarity index 92%
rename from
paimon-vector/paimon-vector-index/src/test/java/org/apache/paimon/vector/index/SeekableStreamVectorIndexInputTest.java
rename to
paimon-vector/src/test/java/org/apache/paimon/vector/index/SeekableStreamVectorIndexInputTest.java
index ddb29eab6e..82eeee198e 100644
---
a/paimon-vector/paimon-vector-index/src/test/java/org/apache/paimon/vector/index/SeekableStreamVectorIndexInputTest.java
+++
b/paimon-vector/src/test/java/org/apache/paimon/vector/index/SeekableStreamVectorIndexInputTest.java
@@ -30,15 +30,15 @@ import java.util.concurrent.atomic.AtomicInteger;
import static org.assertj.core.api.Assertions.assertThat;
-/** Tests for {@link VectorGlobalIndexReader.SeekableStreamVectorIndexInput}.
*/
+/** Tests for {@link
NativeVectorGlobalIndexReader.SeekableStreamVectorIndexInput}. */
public class SeekableStreamVectorIndexInputTest {
@Test
public void testVectoredReadableInputUsesParallelPositionReads() throws
Exception {
byte[] data = data(128 * 1024);
TestVectoredSeekableInputStream input = new
TestVectoredSeekableInputStream(data, 2);
- VectorGlobalIndexReader.SeekableStreamVectorIndexInput indexInput =
- new
VectorGlobalIndexReader.SeekableStreamVectorIndexInput(input);
+ NativeVectorGlobalIndexReader.SeekableStreamVectorIndexInput
indexInput =
+ new
NativeVectorGlobalIndexReader.SeekableStreamVectorIndexInput(input);
byte[][] buffers = new byte[][] {new byte[64], new byte[64]};
indexInput.pread(new long[] {0, 32 * 1024}, buffers);
@@ -54,8 +54,8 @@ public class SeekableStreamVectorIndexInputTest {
public void testFallbackToSequentialReadWhenRangesOverlap() {
byte[] data = data(1024);
TestVectoredSeekableInputStream input = new
TestVectoredSeekableInputStream(data, 0);
- VectorGlobalIndexReader.SeekableStreamVectorIndexInput indexInput =
- new
VectorGlobalIndexReader.SeekableStreamVectorIndexInput(input);
+ NativeVectorGlobalIndexReader.SeekableStreamVectorIndexInput
indexInput =
+ new
NativeVectorGlobalIndexReader.SeekableStreamVectorIndexInput(input);
byte[][] buffers = new byte[][] {new byte[64], new byte[64]};
indexInput.pread(new long[] {0, 32}, buffers);
