This is an automated email from the ASF dual-hosted git repository. JingsongLi pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/paimon-rust.git
The following commit(s) were added to refs/heads/main by this push:
new f8471ea4 feat(ftindex): read Java full-text archives via
paimon-ftindex-core (#563)
f8471ea4 is described below
commit f8471ea417189823fe90a6161b077c9c6c10864e
Author: Junrui Lee <[email protected]>
AuthorDate: Wed Jul 22 21:58:54 2026 +0800
feat(ftindex): read Java full-text archives via paimon-ftindex-core (#563)
---
Cargo.lock | 510 ++++++++++++++++++++-
DEPENDENCIES.rust.tsv | 41 ++
bindings/python/DEPENDENCIES.rust.tsv | 44 ++
.../integrations/datafusion/DEPENDENCIES.rust.tsv | 42 ++
crates/paimon/Cargo.toml | 3 +-
crates/paimon/DEPENDENCIES.rust.tsv | 41 ++
crates/paimon/src/ftindex/mod.rs | 29 ++
crates/paimon/src/ftindex/reader.rs | 282 ++++++++++++
crates/paimon/src/lib.rs | 2 +
9 files changed, 970 insertions(+), 24 deletions(-)
diff --git a/Cargo.lock b/Cargo.lock
index 43181baf..126dfeb1 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -8,6 +8,12 @@ version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"
+[[package]]
+name = "adler32"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234"
+
[[package]]
name = "aes"
version = "0.8.4"
@@ -935,6 +941,12 @@ version = "3.20.3"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649"
+[[package]]
+name = "bytecount"
+version = "0.6.9"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "175812e0be2bccb6abe50bb8d566126198344f707e304f45c648fd8f2cc0365e"
+
[[package]]
name = "bytemuck"
version = "1.25.2"
@@ -983,6 +995,15 @@ dependencies = [
"shlex 2.0.1",
]
+[[package]]
+name = "cedarwood"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "c0524a528a6a0288df1863c3c20fe92c301875b4941e7b6c4b394ab08c5a4c55"
+dependencies = [
+ "smallvec",
+]
+
[[package]]
name = "census"
version = "0.4.2"
@@ -1042,7 +1063,7 @@ source =
"registry+https://github.com/rust-lang/crates.io-index";
checksum = "a6139a8597ed92cf816dfb33f5dd6cf0bb93a6adc938f11039f371bc5bcd26c3"
dependencies = [
"chrono",
- "phf",
+ "phf 0.12.1",
]
[[package]]
@@ -1449,6 +1470,12 @@ dependencies = [
"syn 2.0.119",
]
+[[package]]
+name = "dary_heap"
+version = "0.3.9"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "8b1e3a325bc115f096c8b77bbf027a7c2592230e70be2d985be950d3d5e60ebe"
+
[[package]]
name = "dashmap"
version = "6.2.1"
@@ -2191,6 +2218,12 @@ dependencies = [
"sqlparser",
]
+[[package]]
+name = "datasketches"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "c286de4e81ea2590afc24d754e0f83810c566f50a1388fa75ebd57928c0d9745"
+
[[package]]
name = "defmt"
version = "1.1.1"
@@ -2319,6 +2352,12 @@ version = "1.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "75b325c5dbd37f80359721ad39aca5a29fb04c89279657cffdda8736d0c0b9d2"
+[[package]]
+name = "downcast-rs"
+version = "2.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "117240f60069e65410b3ae1bb213295bd828f707b5bec6596a1afc8793ce0cbc"
+
[[package]]
name = "dtype_dispatch"
version = "0.2.1"
@@ -2378,6 +2417,17 @@ version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f"
+[[package]]
+name = "erased-serde"
+version = "0.4.10"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "d2add8a07dd6a8d93ff627029c51de145e12686fbc36ecb298ac22e74cf02dec"
+dependencies = [
+ "serde",
+ "serde_core",
+ "typeid",
+]
+
[[package]]
name = "errno"
version = "0.3.14"
@@ -2564,6 +2614,16 @@ dependencies = [
"windows-sys 0.52.0",
]
+[[package]]
+name = "fs4"
+version = "0.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "8640e34b88f7652208ce9e88b1a37a2ae95227d84abec377ccd3c5cfeb141ed4"
+dependencies = [
+ "rustix 1.1.4",
+ "windows-sys 0.59.0",
+]
+
[[package]]
name = "fs_extra"
version = "1.3.0"
@@ -2827,6 +2887,17 @@ dependencies = [
"foldhash 0.1.5",
]
+[[package]]
+name = "hashbrown"
+version = "0.16.1"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100"
+dependencies = [
+ "allocator-api2",
+ "equivalent",
+ "foldhash 0.2.0",
+]
+
[[package]]
name = "hashbrown"
version = "0.17.1"
@@ -3201,6 +3272,39 @@ dependencies = [
"icu_properties",
]
+[[package]]
+name = "include-flate"
+version = "0.3.4"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "48f173716febb1ad596c16ea5637b5f1790ea32de8e627493ff82bc73b0876ce"
+dependencies = [
+ "include-flate-codegen",
+ "include-flate-compress",
+]
+
+[[package]]
+name = "include-flate-codegen"
+version = "0.3.4"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "4a7875b62a72ad3f3203cdd8950d4cf9947db036030b974b8b37ceae90c8d8c0"
+dependencies = [
+ "include-flate-compress",
+ "proc-macro-error3",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.119",
+]
+
+[[package]]
+name = "include-flate-compress"
+version = "0.3.4"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "44fbb9c5ccb9a5b67b4afa2974c27e5507ea1bf6d22828cef418e4dfaeca51dd"
+dependencies = [
+ "libflate",
+ "zstd",
+]
+
[[package]]
name = "indexmap"
version = "1.9.3"
@@ -3306,6 +3410,30 @@ version = "1.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
+[[package]]
+name = "jieba-macros"
+version = "0.10.3"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "34904340bc65749a9e9a02fcc7f3368e675427c18447b9bbe02df52c15c9a36a"
+dependencies = [
+ "phf_codegen",
+]
+
+[[package]]
+name = "jieba-rs"
+version = "0.10.3"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "bb5bdea4dc241d589e179f39d2a778f31490f3370aa2f626223dbd930ebc5c9d"
+dependencies = [
+ "bytecount",
+ "cedarwood",
+ "include-flate",
+ "jieba-macros",
+ "phf 0.13.1",
+ "regex",
+ "rustc-hash 2.1.3",
+]
+
[[package]]
name = "jiff"
version = "0.2.34"
@@ -3560,6 +3688,30 @@ version = "0.2.186"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66"
+[[package]]
+name = "libflate"
+version = "2.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "cd96e993e5f3368b0cb8497dae6c860c22af8ff18388c61c6c0b86c58d86b5df"
+dependencies = [
+ "adler32",
+ "crc32fast",
+ "dary_heap",
+ "libflate_lz77",
+ "no_std_io2",
+]
+
+[[package]]
+name = "libflate_lz77"
+version = "2.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "ff7a10e427698aef6eef269482776debfef63384d30f13aad39a1a95e0e098fd"
+dependencies = [
+ "hashbrown 0.16.1",
+ "no_std_io2",
+ "rle-decode-fast",
+]
+
[[package]]
name = "libloading"
version = "0.8.9"
@@ -3660,6 +3812,15 @@ dependencies = [
"hashbrown 0.15.5",
]
+[[package]]
+name = "lru"
+version = "0.16.4"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "7f66e8d5d03f609abc3a39e6f08e4164ebf1447a732906d39eb9b99b7919ef39"
+dependencies = [
+ "hashbrown 0.16.1",
+]
+
[[package]]
name = "lru-slab"
version = "0.1.2"
@@ -3765,6 +3926,15 @@ dependencies = [
"log",
]
+[[package]]
+name = "measure_time"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "51c55d61e72fc3ab704396c5fa16f4c184db37978ae4e94ca8959693a235fc0e"
+dependencies = [
+ "log",
+]
+
[[package]]
name = "memchr"
version = "2.8.3"
@@ -3889,6 +4059,15 @@ version = "6.6.666"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "cf5a574dadd7941adeaa71823ecba5e28331b8313fb2e1c6a5c7e5981ea53ad6"
+[[package]]
+name = "no_std_io2"
+version = "0.9.4"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "418abd1b6d34fbf6cae440dc874771b0525a604428704c76e48b29a5e67b8003"
+dependencies = [
+ "memchr",
+]
+
[[package]]
name = "nom"
version = "7.1.3"
@@ -4380,6 +4559,15 @@ dependencies = [
"num-traits",
]
+[[package]]
+name = "ordered-float"
+version = "5.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "b7d950ca161dc355eaf28f82b11345ed76c6e1f6eb1f4f4479e0323b9e2fbd0e"
+dependencies = [
+ "num-traits",
+]
+
[[package]]
name = "ordered-multimap"
version = "0.7.3"
@@ -4399,6 +4587,15 @@ dependencies = [
"stable_deref_trait",
]
+[[package]]
+name = "ownedbytes"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "2fbd56f7631767e61784dc43f8580f403f4475bd4aaa4da003e6295e1bab4a7e"
+dependencies = [
+ "stable_deref_trait",
+]
+
[[package]]
name = "paimon"
version = "0.3.0"
@@ -4440,6 +4637,7 @@ dependencies = [
"opendal-service-oss",
"opendal-service-s3",
"orc-rust",
+ "paimon-ftindex-core",
"paimon-mosaic-core",
"paimon-vindex-core",
"parquet",
@@ -4457,7 +4655,7 @@ dependencies = [
"sha2 0.10.9",
"snafu 0.9.1",
"snap",
- "tantivy",
+ "tantivy 0.22.1",
"tempfile",
"tokio",
"tokio-util",
@@ -4509,6 +4707,24 @@ dependencies = [
"uuid",
]
+[[package]]
+name = "paimon-ftindex-core"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "3c9b761618093284c768eacaa2a93c965c697c60ae1d323b55b44de35a59ae12"
+dependencies = [
+ "levenshtein_automata",
+ "roaring",
+ "serde",
+ "serde_json",
+ "tantivy 0.26.1",
+ "tantivy-common 0.11.0",
+ "tantivy-fst",
+ "tantivy-jieba",
+ "tempfile",
+ "thiserror 1.0.69",
+]
+
[[package]]
name = "paimon-integration-tests"
version = "0.3.0"
@@ -4716,7 +4932,37 @@ version = "0.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "913273894cec178f401a31ec4b656318d95473527be05c0752cc41cdc32be8b7"
dependencies = [
- "phf_shared",
+ "phf_shared 0.12.1",
+]
+
+[[package]]
+name = "phf"
+version = "0.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf"
+dependencies = [
+ "phf_shared 0.13.1",
+ "serde",
+]
+
+[[package]]
+name = "phf_codegen"
+version = "0.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "49aa7f9d80421bca176ca8dbfebe668cc7a2684708594ec9f3c0db0805d5d6e1"
+dependencies = [
+ "phf_generator",
+ "phf_shared 0.13.1",
+]
+
+[[package]]
+name = "phf_generator"
+version = "0.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "135ace3a761e564ec88c03a77317a7c6b80bb7f7135ef2544dbe054243b89737"
+dependencies = [
+ "fastrand",
+ "phf_shared 0.13.1",
]
[[package]]
@@ -4728,6 +4974,15 @@ dependencies = [
"siphasher",
]
+[[package]]
+name = "phf_shared"
+version = "0.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "e57fef6bc5981e38c2ce2d63bfa546861309f875b8a75f092d1d54ae2d64f266"
+dependencies = [
+ "siphasher",
+]
+
[[package]]
name = "pin-project"
version = "1.1.13"
@@ -4903,6 +5158,28 @@ dependencies = [
"toml_edit",
]
+[[package]]
+name = "proc-macro-error-attr3"
+version = "3.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "34e4dd828515431dd6c4a030d26f7eaed7dd4778226e9d2bb968d65ca4ec3d4d"
+dependencies = [
+ "proc-macro2",
+ "quote",
+]
+
+[[package]]
+name = "proc-macro-error3"
+version = "3.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "5ee475e440453418ff1335189eddf7101ba502cd818ab7ae04209bc83aa925aa"
+dependencies = [
+ "proc-macro-error-attr3",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.119",
+]
+
[[package]]
name = "proc-macro2"
version = "1.0.107"
@@ -5609,6 +5886,12 @@ dependencies = [
"windows-sys 0.52.0",
]
+[[package]]
+name = "rle-decode-fast"
+version = "1.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "3582f63211428f83597b51b2ddb88e2a91a9d52d12831f9d08f5e624e8977422"
+
[[package]]
name = "roaring"
version = "0.11.4"
@@ -6171,6 +6454,9 @@ name = "sketches-ddsketch"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "05e40b6cf54d988dc1a2223531b969c9a9e30906ad90ef64890c27b4bfbb46ea"
+dependencies = [
+ "serde",
+]
[[package]]
name = "slab"
@@ -6486,17 +6772,17 @@ dependencies = [
"census",
"crc32fast",
"crossbeam-channel",
- "downcast-rs",
+ "downcast-rs 1.2.1",
"fastdivide",
"fnv",
- "fs4",
+ "fs4 0.8.4",
"htmlescape",
"itertools 0.12.1",
"levenshtein_automata",
"log",
- "lru",
+ "lru 0.12.5",
"lz4_flex 0.11.6",
- "measure_time",
+ "measure_time 0.8.3",
"memmap2",
"num_cpus",
"once_cell",
@@ -6509,13 +6795,13 @@ dependencies = [
"serde_json",
"sketches-ddsketch 0.2.2",
"smallvec",
- "tantivy-bitpacker",
- "tantivy-columnar",
- "tantivy-common",
+ "tantivy-bitpacker 0.6.0",
+ "tantivy-columnar 0.3.0",
+ "tantivy-common 0.7.0",
"tantivy-fst",
- "tantivy-query-grammar",
- "tantivy-stacker",
- "tantivy-tokenizer-api",
+ "tantivy-query-grammar 0.22.0",
+ "tantivy-stacker 0.3.0",
+ "tantivy-tokenizer-api 0.3.0",
"tempfile",
"thiserror 1.0.69",
"time",
@@ -6523,6 +6809,59 @@ dependencies = [
"winapi",
]
+[[package]]
+name = "tantivy"
+version = "0.26.1"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "edde6a10743fff00a4e1a8c9ef020bf5f3cbad301b7d2d39f2b07f123c4eac07"
+dependencies = [
+ "aho-corasick",
+ "arc-swap",
+ "base64",
+ "bitpacking",
+ "bon",
+ "byteorder",
+ "census",
+ "crc32fast",
+ "crossbeam-channel",
+ "datasketches",
+ "downcast-rs 2.0.2",
+ "fastdivide",
+ "fnv",
+ "fs4 0.13.1",
+ "htmlescape",
+ "itertools 0.14.0",
+ "levenshtein_automata",
+ "log",
+ "lru 0.16.4",
+ "lz4_flex 0.13.1",
+ "measure_time 0.9.0",
+ "memmap2",
+ "once_cell",
+ "oneshot 0.1.13",
+ "rayon",
+ "regex",
+ "rust-stemmers",
+ "rustc-hash 2.1.3",
+ "serde",
+ "serde_json",
+ "sketches-ddsketch 0.4.0",
+ "smallvec",
+ "tantivy-bitpacker 0.10.0",
+ "tantivy-columnar 0.7.0",
+ "tantivy-common 0.11.0",
+ "tantivy-fst",
+ "tantivy-query-grammar 0.26.0",
+ "tantivy-stacker 0.7.0",
+ "tantivy-tokenizer-api 0.7.0",
+ "tempfile",
+ "thiserror 2.0.19",
+ "time",
+ "typetag",
+ "uuid",
+ "winapi",
+]
+
[[package]]
name = "tantivy-bitpacker"
version = "0.6.0"
@@ -6532,20 +6871,45 @@ dependencies = [
"bitpacking",
]
+[[package]]
+name = "tantivy-bitpacker"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "4fed3d674429bcd2de5d0a6d1aa5495fed8afd9c5ecce993019caf7615f53fa4"
+dependencies = [
+ "bitpacking",
+]
+
[[package]]
name = "tantivy-columnar"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "12722224ffbe346c7fec3275c699e508fd0d4710e629e933d5736ec524a1f44e"
dependencies = [
- "downcast-rs",
+ "downcast-rs 1.2.1",
"fastdivide",
"itertools 0.12.1",
"serde",
- "tantivy-bitpacker",
- "tantivy-common",
- "tantivy-sstable",
- "tantivy-stacker",
+ "tantivy-bitpacker 0.6.0",
+ "tantivy-common 0.7.0",
+ "tantivy-sstable 0.3.0",
+ "tantivy-stacker 0.3.0",
+]
+
+[[package]]
+name = "tantivy-columnar"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "c57166f5bcfd478f370ab8445afb4678dce44801fa5ce5c451aaf8595583c5dc"
+dependencies = [
+ "downcast-rs 2.0.2",
+ "fastdivide",
+ "itertools 0.14.0",
+ "serde",
+ "tantivy-bitpacker 0.10.0",
+ "tantivy-common 0.11.0",
+ "tantivy-sstable 0.7.0",
+ "tantivy-stacker 0.7.0",
]
[[package]]
@@ -6556,7 +6920,20 @@ checksum =
"8019e3cabcfd20a1380b491e13ff42f57bb38bf97c3d5fa5c07e50816e0621f4"
dependencies = [
"async-trait",
"byteorder",
- "ownedbytes",
+ "ownedbytes 0.7.0",
+ "serde",
+ "time",
+]
+
+[[package]]
+name = "tantivy-common"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "bbf10915aa75da3c3b0d58b58853d2e889efbaf32d4982a4c3715dde6bba23e5"
+dependencies = [
+ "async-trait",
+ "byteorder",
+ "ownedbytes 0.9.0",
"serde",
"time",
]
@@ -6572,6 +6949,17 @@ dependencies = [
"utf8-ranges",
]
+[[package]]
+name = "tantivy-jieba"
+version = "0.20.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "3392170e86f1c387170aba7d171a466ffdc98a8b55b006e19ac64b123a7b690a"
+dependencies = [
+ "jieba-rs",
+ "lazy_static",
+ "tantivy-tokenizer-api 0.7.0",
+]
+
[[package]]
name = "tantivy-query-grammar"
version = "0.22.0"
@@ -6581,14 +6969,41 @@ dependencies = [
"nom",
]
+[[package]]
+name = "tantivy-query-grammar"
+version = "0.26.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "dfadb8526b6da90704feb293b0701a6aae62ea14983143344be2dc5ce30f1d82"
+dependencies = [
+ "fnv",
+ "nom",
+ "ordered-float 5.3.0",
+ "serde",
+ "serde_json",
+]
+
[[package]]
name = "tantivy-sstable"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "c69578242e8e9fc989119f522ba5b49a38ac20f576fc778035b96cc94f41f98e"
dependencies = [
- "tantivy-bitpacker",
- "tantivy-common",
+ "tantivy-bitpacker 0.6.0",
+ "tantivy-common 0.7.0",
+ "tantivy-fst",
+ "zstd",
+]
+
+[[package]]
+name = "tantivy-sstable"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "8a2cfc3ac5164cbadc28965ffb145a8f47582a60ae5897859ad8d4316596c606"
+dependencies = [
+ "futures-util",
+ "itertools 0.14.0",
+ "tantivy-bitpacker 0.10.0",
+ "tantivy-common 0.11.0",
"tantivy-fst",
"zstd",
]
@@ -6601,7 +7016,17 @@ checksum =
"c56d6ff5591fc332739b3ce7035b57995a3ce29a93ffd6012660e0949c956ea8"
dependencies = [
"murmurhash32",
"rand_distr",
- "tantivy-common",
+ "tantivy-common 0.7.0",
+]
+
+[[package]]
+name = "tantivy-stacker"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "6cbb051742da9d53ca9e8fff43a9b10e319338b24e2c0e15d0372df19ffeb951"
+dependencies = [
+ "murmurhash32",
+ "tantivy-common 0.11.0",
]
[[package]]
@@ -6613,6 +7038,15 @@ dependencies = [
"serde",
]
+[[package]]
+name = "tantivy-tokenizer-api"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "eac258c2c6390673f2685813afeeafcb8c4e0ee7de8dd3fc46838dcc37263f98"
+dependencies = [
+ "serde",
+]
+
[[package]]
name = "tap"
version = "1.0.1"
@@ -6703,7 +7137,7 @@ checksum =
"7e54bc85fc7faa8bc175c4bab5b92ba8d9a3ce893d0e9f42cc455c8ab16a9e09"
dependencies = [
"byteorder",
"integer-encoding",
- "ordered-float",
+ "ordered-float 2.10.1",
]
[[package]]
@@ -6983,12 +7417,42 @@ dependencies = [
"syn 2.0.119",
]
+[[package]]
+name = "typeid"
+version = "1.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "bc7d623258602320d5c55d1bc22793b57daff0ec7efc270ea7d55ce1d5f5471c"
+
[[package]]
name = "typenum"
version = "1.20.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20"
+[[package]]
+name = "typetag"
+version = "0.2.23"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "c90e86058a30d42a1a928dfb4b49bb33c98c3a2b4909492e6b0881cd94798ec2"
+dependencies = [
+ "erased-serde",
+ "inventory",
+ "once_cell",
+ "serde",
+ "typetag-impl",
+]
+
+[[package]]
+name = "typetag-impl"
+version = "0.2.23"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "f153acc4e99a5f2a5aefa09fb078be54e26271b2813f6041200b224c098d8328"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 3.0.2",
+]
+
[[package]]
name = "unicode-ident"
version = "1.0.24"
diff --git a/DEPENDENCIES.rust.tsv b/DEPENDENCIES.rust.tsv
index 7c66bcb0..166d7ce6 100644
--- a/DEPENDENCIES.rust.tsv
+++ b/DEPENDENCIES.rust.tsv
@@ -1,5 +1,6 @@
crate 0BSD Apache-2.0 Apache-2.0 WITH LLVM-exception BSD-2-Clause
BSD-3-Clause BSL-1.0 CC0-1.0 CDLA-Permissive-2.0 EPL-2.0 ISC
LGPL-2.1-or-later MIT MIT-0 MPL-2.0 Unicode-3.0 Unlicense
Zlib bzip2-1.0.6 zlib-acknowledgement
[email protected] X X
X
[email protected]
X
[email protected] X
X
[email protected] X
X
[email protected]
X X
@@ -77,12 +78,14 @@ [email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X X
[email protected]
X X
[email protected]
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
[email protected]
X
[email protected] X
X
[email protected] X
X
@@ -134,6 +137,7 @@ [email protected] X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected] X
X
[email protected]
X
[email protected] X
[email protected] X
@@ -169,6 +173,7 @@ [email protected] X
[email protected] X
[email protected] X
[email protected] X
[email protected] X
[email protected] X
X
[email protected] X
X
[email protected] X
X
@@ -179,6 +184,7 @@ [email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
[email protected] X X
X
[email protected] X
X
@@ -186,6 +192,7 @@ [email protected] X X
X
[email protected] X
[email protected] X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
@@ -206,6 +213,7 @@ [email protected]
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected]
X
[email protected] X
@@ -232,6 +240,7 @@ [email protected]
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
[email protected] X
X
@@ -264,6 +273,9 @@ [email protected]
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
[email protected] X
[email protected] X
[email protected] X
X
[email protected] X
X
[email protected] X
@@ -275,6 +287,8 @@ [email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected]
X
[email protected]
X
[email protected]
X X
[email protected]
X X
[email protected]
X X
@@ -299,6 +313,8 @@ [email protected] X
X
[email protected] X
X
[email protected]
X
[email protected] X
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected] X
X
@@ -311,6 +327,7 @@ [email protected]
X
[email protected] X
X
[email protected] X
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected]
X
@@ -322,6 +339,7 @@ [email protected] X
X
[email protected] X
X
[email protected] X
[email protected]
X
[email protected]
X
[email protected]
X X
[email protected] X
X
[email protected] X
X
@@ -334,6 +352,7 @@ [email protected] X
[email protected] X
[email protected] X
X
[email protected] X
X
X
[email protected] X
X
[email protected]
X
[email protected] X
X X
[email protected] X
X
X
@@ -372,11 +391,14 @@ [email protected] X
X
[email protected]
X
[email protected] X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected] X
[email protected] X
[email protected] X
[email protected] X
[email protected] X
[email protected] X
[email protected] X
@@ -394,7 +416,11 @@ [email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
@@ -414,6 +440,8 @@ [email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
[email protected] X
@@ -464,6 +492,7 @@ [email protected] X
[email protected] X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
@@ -551,14 +580,23 @@ [email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected]
X X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected] X
[email protected] X
X
@@ -592,7 +630,10 @@ [email protected]
X
[email protected]
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X X
[email protected] X
X
[email protected] X
X
diff --git a/bindings/python/DEPENDENCIES.rust.tsv
b/bindings/python/DEPENDENCIES.rust.tsv
index 76426cb4..041777ad 100644
--- a/bindings/python/DEPENDENCIES.rust.tsv
+++ b/bindings/python/DEPENDENCIES.rust.tsv
@@ -1,5 +1,6 @@
crate 0BSD Apache-2.0 Apache-2.0 WITH LLVM-exception BSD-2-Clause
BSD-3-Clause BSL-1.0 CC0-1.0 CDLA-Permissive-2.0 EPL-2.0 ISC
LGPL-2.1-or-later MIT MIT-0 MPL-2.0 Unicode-3.0 Unlicense
Zlib bzip2-1.0.6 zlib-acknowledgement
[email protected] X X
X
[email protected]
X
[email protected] X
X
[email protected] X
X
[email protected]
X X
@@ -54,12 +55,14 @@ [email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X X
[email protected]
X X
[email protected]
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
[email protected]
X
[email protected] X
X
[email protected] X
X
@@ -99,6 +102,7 @@ [email protected] X
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected] X
X
[email protected]
X
[email protected] X
[email protected] X
@@ -134,6 +138,7 @@ [email protected] X
[email protected] X
[email protected] X
[email protected] X
[email protected] X
[email protected] X
X
[email protected] X
X
[email protected] X
X
@@ -144,10 +149,12 @@ [email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X X
X
[email protected] X
X
[email protected] X X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected]
X
X
@@ -162,6 +169,7 @@ [email protected]
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected]
X
[email protected] X
X
@@ -185,6 +193,7 @@ [email protected]
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
[email protected] X
X
@@ -216,14 +225,20 @@ [email protected]
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
[email protected] X
[email protected] X
[email protected] X
X
[email protected] X
X
[email protected] X
[email protected]
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected]
X
[email protected]
X
[email protected]
X X
[email protected]
X X
[email protected]
X X
@@ -244,6 +259,8 @@ [email protected] X
X
[email protected] X
X
[email protected]
X
[email protected] X
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected] X
X
[email protected] X
X
@@ -255,6 +272,7 @@ [email protected]
X
[email protected] X
X
[email protected] X
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected]
X
@@ -263,6 +281,7 @@ [email protected] X
X
[email protected] X
X
[email protected] X
[email protected]
X
[email protected]
X
[email protected]
X X
[email protected] X
X
[email protected] X
X
@@ -273,6 +292,7 @@ [email protected]
X
[email protected] X
[email protected] X
[email protected] X
X
[email protected] X
X
[email protected]
X
[email protected] X
X
[email protected] X
X
@@ -305,10 +325,13 @@ [email protected] X
X
[email protected]
X
[email protected] X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected] X
[email protected] X
[email protected] X
[email protected] X
[email protected] X
[email protected] X
X
@@ -321,7 +344,11 @@ [email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
@@ -338,6 +365,8 @@ [email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
[email protected] X
@@ -384,12 +413,14 @@ [email protected] X
[email protected] X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected]
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X X
X
[email protected] X X
X
@@ -433,6 +464,7 @@ [email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
[email protected] X
[email protected]
X
[email protected] X
X
[email protected] X
X
@@ -462,14 +494,23 @@ [email protected]
X
[email protected] X
X
[email protected] X
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected]
X X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected] X
[email protected] X
X
[email protected] X
X
@@ -501,7 +542,10 @@ [email protected]
X
[email protected]
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X X
[email protected] X
X
[email protected] X
X
diff --git a/crates/integrations/datafusion/DEPENDENCIES.rust.tsv
b/crates/integrations/datafusion/DEPENDENCIES.rust.tsv
index 7a3621e9..f230597d 100644
--- a/crates/integrations/datafusion/DEPENDENCIES.rust.tsv
+++ b/crates/integrations/datafusion/DEPENDENCIES.rust.tsv
@@ -1,5 +1,6 @@
crate 0BSD Apache-2.0 Apache-2.0 WITH LLVM-exception BSD-2-Clause
BSD-3-Clause BSL-1.0 CC0-1.0 CDLA-Permissive-2.0 ISC
LGPL-2.1-or-later MIT MIT-0 MPL-2.0 Unicode-3.0 Unlicense
Zlib bzip2-1.0.6 zlib-acknowledgement
[email protected] X X
X
[email protected]
X
[email protected] X
X
[email protected]
X X
[email protected] X
@@ -64,11 +65,13 @@ [email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X X
[email protected]
X X
[email protected]
X
[email protected] X
X
[email protected] X
X
[email protected] X
[email protected]
X
[email protected] X
X
[email protected] X
X
@@ -109,6 +112,7 @@ [email protected] X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected] X
X
[email protected]
X
[email protected] X
[email protected] X
@@ -141,6 +145,7 @@ [email protected] X
[email protected] X
[email protected] X
[email protected] X
[email protected] X
[email protected] X
X
[email protected] X
X
[email protected] X
X
@@ -148,6 +153,7 @@ [email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
[email protected] X X
X
[email protected] X
X
@@ -155,6 +161,7 @@ [email protected] X X
X
[email protected] X
[email protected] X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
@@ -175,6 +182,7 @@ [email protected]
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected]
X
[email protected] X
@@ -198,6 +206,7 @@ [email protected]
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
@@ -229,6 +238,9 @@ [email protected]
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
[email protected] X
[email protected] X
[email protected] X
X
[email protected] X
[email protected]
X
@@ -238,6 +250,8 @@ [email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected]
X
[email protected]
X
[email protected]
X X
[email protected]
X X
[email protected]
X X
@@ -250,6 +264,7 @@ [email protected] X
X
[email protected] X
X
[email protected]
X
[email protected] X
X
[email protected] X
X
[email protected] X
X X
[email protected] X
X
X
[email protected]
X
@@ -261,6 +276,8 @@ [email protected] X
X
[email protected] X
X
[email protected]
X
[email protected] X
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected] X
X
@@ -272,6 +289,7 @@ [email protected]
X
[email protected] X
X
[email protected] X
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected]
X
@@ -282,6 +300,7 @@ [email protected] X
X
[email protected] X
X
[email protected] X
[email protected]
X
[email protected]
X
[email protected]
X X
[email protected] X
X
[email protected] X
X
@@ -294,6 +313,7 @@ [email protected] X
[email protected] X
[email protected] X
X
[email protected] X
X X
[email protected] X
X
[email protected]
X
[email protected] X
X X
[email protected] X
X X
@@ -323,10 +343,13 @@ [email protected] X
X
[email protected]
X
[email protected] X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected] X
[email protected] X
[email protected] X
[email protected] X
[email protected] X
[email protected] X
X
@@ -338,7 +361,11 @@ [email protected] X
[email protected] X
X
[email protected] X
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
@@ -353,6 +380,8 @@ [email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
[email protected] X
@@ -391,6 +420,7 @@ [email protected] X
[email protected] X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected]
X
[email protected] X
X
@@ -465,14 +495,23 @@ [email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected]
X X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected] X
X
[email protected]
X
@@ -502,7 +541,10 @@ [email protected]
X
[email protected]
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X X
[email protected] X
X
[email protected] X
X
diff --git a/crates/paimon/Cargo.toml b/crates/paimon/Cargo.toml
index f8bb188b..a2042c5a 100644
--- a/crates/paimon/Cargo.toml
+++ b/crates/paimon/Cargo.toml
@@ -42,7 +42,7 @@ storage-all = [
"storage-gcs",
"storage-hdfs",
]
-fulltext = ["tantivy", "tempfile"]
+fulltext = ["tantivy", "tempfile", "dep:paimon-ftindex-core"]
vortex = ["dep:vortex"]
storage-memory = ["opendal/services-memory"]
@@ -114,6 +114,7 @@ uuid = { version = "1", features = ["v4"] }
urlencoding = "2.1"
tantivy = { version = "0.22", optional = true }
tempfile = { version = "3", optional = true }
+paimon-ftindex-core = { version = "0.1.0", optional = true }
paimon-mosaic-core = "0.2.0"
paimon-vindex-core = "0.2.0"
vortex = { version = "0.75.0", features = ["tokio"], optional = true }
diff --git a/crates/paimon/DEPENDENCIES.rust.tsv
b/crates/paimon/DEPENDENCIES.rust.tsv
index b60dd8de..b032f87b 100644
--- a/crates/paimon/DEPENDENCIES.rust.tsv
+++ b/crates/paimon/DEPENDENCIES.rust.tsv
@@ -1,5 +1,6 @@
crate 0BSD Apache-2.0 Apache-2.0 WITH LLVM-exception BSD-2-Clause
BSD-3-Clause BSL-1.0 CC0-1.0 CDLA-Permissive-2.0 ISC
LGPL-2.1-or-later MIT MIT-0 MPL-2.0 Unicode-3.0 Unlicense
Zlib zlib-acknowledgement
[email protected] X X
X
[email protected]
X
[email protected] X
X
[email protected] X
X
[email protected]
X X
@@ -61,11 +62,13 @@ [email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X X
[email protected]
X X
[email protected]
X
[email protected] X
X
[email protected] X
X
[email protected] X
[email protected]
X
[email protected] X
X
[email protected] X
X
@@ -109,7 +112,9 @@ [email protected] X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected] X
X
[email protected]
X
[email protected] X
[email protected] X
X
[email protected] X
X
[email protected] X
X
@@ -120,6 +125,7 @@ [email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
[email protected] X X
X
[email protected] X
X
@@ -127,6 +133,7 @@ [email protected] X X
X
[email protected] X
[email protected] X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
@@ -146,6 +153,7 @@ [email protected]
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected]
X
[email protected] X
@@ -172,6 +180,7 @@ [email protected]
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
[email protected] X
X
@@ -203,6 +212,9 @@ [email protected]
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
[email protected] X
[email protected] X
[email protected] X
X
[email protected] X
X
[email protected] X
@@ -213,6 +225,8 @@ [email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected]
X
[email protected]
X
[email protected]
X X
[email protected]
X X
[email protected]
X X
@@ -236,6 +250,8 @@ [email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected]
X
@@ -246,6 +262,7 @@ [email protected]
X
[email protected] X
X
[email protected] X
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected]
X
@@ -256,6 +273,7 @@ [email protected] X
X
[email protected] X
X
[email protected] X
[email protected]
X
[email protected]
X
[email protected]
X X
[email protected] X
X
[email protected] X
X
@@ -268,6 +286,7 @@ [email protected] X
[email protected] X
[email protected] X
X
[email protected] X
X X
[email protected] X
X
[email protected]
X
[email protected] X
X X
[email protected] X
X X
@@ -303,9 +322,12 @@ [email protected] X
X
[email protected]
X
[email protected] X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected] X
[email protected] X
[email protected] X
[email protected] X
[email protected] X
X
@@ -319,7 +341,11 @@ [email protected]
X
[email protected] X
X
[email protected] X
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
@@ -336,6 +362,8 @@ [email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
[email protected] X
@@ -377,6 +405,7 @@ [email protected] X
[email protected] X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
@@ -456,14 +485,23 @@ [email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected]
X X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected]
X
[email protected] X
X
[email protected]
X
@@ -492,7 +530,10 @@ [email protected]
X
[email protected]
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X
[email protected] X
X X
[email protected] X
X
[email protected] X
X
diff --git a/crates/paimon/src/ftindex/mod.rs b/crates/paimon/src/ftindex/mod.rs
new file mode 100644
index 00000000..0a56c9f1
--- /dev/null
+++ b/crates/paimon/src/ftindex/mod.rs
@@ -0,0 +1,29 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Full-text index reader backed by the shared `paimon-ftindex-core` engine
+//! (the same native core the Java/Python bindings use), reading the v1
+//! archive format.
+
+pub mod reader;
+
+// Re-export the core I/O types that appear in this module's public API so that
+// consumers can implement `SeekRead` (e.g. a remote range-reader) or name the
+// in-memory reader while depending only on `paimon`, without pulling in
+// `paimon-ftindex-core` directly.
+pub use paimon_ftindex_core::io::{ReadRequest, SeekRead, SliceReader};
+pub use reader::{BytesReader, FullTextArchiveReader, FullTextHits};
diff --git a/crates/paimon/src/ftindex/reader.rs
b/crates/paimon/src/ftindex/reader.rs
new file mode 100644
index 00000000..c7dda64d
--- /dev/null
+++ b/crates/paimon/src/ftindex/reader.rs
@@ -0,0 +1,282 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Reader over the `paimon-ftindex-core` v1 archive format.
+
+use std::io;
+
+use bytes::Bytes;
+use paimon_ftindex_core::io::{ReadRequest, SeekRead};
+use paimon_ftindex_core::{FullTextIndexReader, FullTextSearchResult};
+use roaring::RoaringTreemap;
+
+use crate::io::InputFile;
+
+/// Search hits from a full-text archive: parallel row-id and score arrays,
+/// ordered best-score-first (as returned by the engine).
+#[derive(Debug, Clone)]
+pub struct FullTextHits {
+ pub row_ids: Vec<i64>,
+ pub scores: Vec<f32>,
+}
+
+impl From<FullTextSearchResult> for FullTextHits {
+ fn from(r: FullTextSearchResult) -> Self {
+ Self {
+ row_ids: r.row_ids,
+ scores: r.scores,
+ }
+ }
+}
+
+/// Reader over a single full-text index archive, backed by the shared
+/// `paimon-ftindex-core` engine (v1 archive format).
+///
+/// Generic over any `SeekRead` implementation, allowing both whole-file
+/// (via `BytesReader`/`SliceReader`) and streaming/range-read strategies.
+pub struct FullTextArchiveReader<R: SeekRead + 'static> {
+ inner: FullTextIndexReader<R>,
+}
+
+impl<R: SeekRead + 'static> FullTextArchiveReader<R> {
+ /// Open a full-text archive reader over any `SeekRead` implementation.
+ ///
+ /// Use this for streaming/range-read strategies (e.g., remote FileIO with
+ /// bounded concurrency). For convenience when reading whole files into
+ /// memory, see [`from_input_file`](Self::from_input_file).
+ pub fn from_seek_read(reader: R) -> crate::Result<Self> {
+ let inner = FullTextIndexReader::open(reader).map_err(map_ft_err)?;
+ Ok(Self { inner })
+ }
+
+ /// Search with a JSON DSL query (see the engine's query spec), returning
up
+ /// to `limit` best-scoring hits.
+ pub fn search(&self, query_json: &str, limit: usize) ->
crate::Result<FullTextHits> {
+ self.inner
+ .search(query_json, limit)
+ .map(Into::into)
+ .map_err(map_ft_err)
+ }
+
+ /// Like [`search`](Self::search) but restricts results to
`include_row_ids`
+ /// (the live-row allow-list). The bitmap is only serialized, never
retained,
+ /// so it is borrowed to spare callers a clone when reusing an allow-list
+ /// across archives. An empty allow-list admits no rows, so this returns
+ /// empty hits immediately without running the query.
+ pub fn search_with_include(
+ &self,
+ query_json: &str,
+ limit: usize,
+ include_row_ids: &RoaringTreemap,
+ ) -> crate::Result<FullTextHits> {
+ if include_row_ids.is_empty() {
+ return Ok(FullTextHits {
+ row_ids: Vec::new(),
+ scores: Vec::new(),
+ });
+ }
+ let mut filter_bytes =
Vec::with_capacity(include_row_ids.serialized_size());
+ include_row_ids
+ .serialize_into(&mut filter_bytes)
+ .map_err(|e| crate::Error::UnexpectedError {
+ message: format!("failed to serialize row-id filter: {e}"),
+ source: Some(Box::new(e)),
+ })?;
+ self.inner
+ .search_with_roaring_filter(query_json, limit, &filter_bytes)
+ .map(Into::into)
+ .map_err(map_ft_err)
+ }
+}
+
+impl FullTextArchiveReader<BytesReader> {
+ /// Read the whole archive from `input` into memory and open the engine
+ /// reader over it. The bytes returned by `input.read()` are held directly
+ /// by a [`BytesReader`], so no second copy is made and peak memory stays
at
+ /// roughly one archive.
+ ///
+ /// This is a convenience wrapper over
[`from_seek_read`](Self::from_seek_read)
+ /// for the common whole-file case. For large or remote archives that
should
+ /// not be fully buffered, call [`from_seek_read`](Self::from_seek_read)
with
+ /// a range-reading [`SeekRead`] implementation instead.
+ pub async fn from_input_file(input: &InputFile) -> crate::Result<Self> {
+ let bytes = input.read().await?;
+ Self::from_seek_read(BytesReader::new(bytes))
+ }
+}
+
+/// A whole-archive [`SeekRead`] backed by an in-memory [`Bytes`] buffer.
+///
+/// Unlike the core `SliceReader` (which owns a `Vec<u8>`), this holds the
+/// `Bytes` returned by `InputFile::read()` directly, so opening a reader from
a
+/// whole-file read does not copy the archive a second time.
+pub struct BytesReader {
+ data: Bytes,
+}
+
+impl BytesReader {
+ /// Wrap an already-read archive buffer.
+ pub fn new(data: Bytes) -> Self {
+ Self { data }
+ }
+}
+
+impl SeekRead for BytesReader {
+ fn pread(&self, ranges: &mut [ReadRequest<'_>]) -> io::Result<()> {
+ for range in ranges {
+ let start = usize::try_from(range.pos)
+ .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput,
"offset overflow"))?;
+ let end = start
+ .checked_add(range.buf.len())
+ .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput,
"range overflow"))?;
+ if end > self.data.len() {
+ return Err(io::Error::new(
+ io::ErrorKind::UnexpectedEof,
+ "read past end of archive",
+ ));
+ }
+ range.buf.copy_from_slice(&self.data[start..end]);
+ }
+ Ok(())
+ }
+}
+
+fn map_ft_err(e: paimon_ftindex_core::FtIndexError) -> crate::Error {
+ crate::Error::UnexpectedError {
+ message: format!("full-text index engine error: {e}"),
+ source: Some(Box::new(e)),
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::io::FileIOBuilder;
+ use paimon_ftindex_core::io::PosWriter;
+ use paimon_ftindex_core::{FullTextIndexConfig, FullTextIndexWriter};
+
+ // Build a tiny archive in memory using the core writer, return its bytes.
+ fn build_archive(docs: &[(i64, &str)]) -> Vec<u8> {
+ let mut writer =
FullTextIndexWriter::new(FullTextIndexConfig::new()).unwrap();
+ for (row_id, text) in docs {
+ writer.add_document(*row_id, (*text).to_string()).unwrap();
+ }
+ let mut out = PosWriter::new(Vec::<u8>::new());
+ writer.write(&mut out).unwrap();
+ out.into_inner()
+ }
+
+ async fn archive_input(bytes: Vec<u8>) -> crate::io::InputFile {
+ let file_io = FileIOBuilder::new("memory").build().unwrap();
+ let path = "memory:/ft-pr1-roundtrip.archive";
+ let output = file_io.new_output(path).unwrap();
+ output.write(Bytes::from(bytes)).await.unwrap();
+ output.to_input_file()
+ }
+
+ #[tokio::test]
+ async fn test_round_trip_search_returns_expected_rows() {
+ let bytes = build_archive(&[
+ (0, "alpha bravo charlie"),
+ (1, "bravo delta"),
+ (2, "echo foxtrot"),
+ ]);
+ let input = archive_input(bytes).await;
+ let reader = FullTextArchiveReader::from_input_file(&input)
+ .await
+ .unwrap();
+
+ let hits = reader.search(r#"{"match":{"query":"bravo"}}"#,
10).unwrap();
+ let mut ids = hits.row_ids.clone();
+ ids.sort_unstable();
+ assert_eq!(ids, vec![0, 1]);
+ assert_eq!(hits.scores.len(), hits.row_ids.len());
+ }
+
+ #[tokio::test]
+ async fn test_search_with_include_restricts_to_allow_list() {
+ let bytes = build_archive(&[
+ (0, "shared token here"),
+ (1, "shared token here"),
+ (2, "shared token here"),
+ ]);
+ let input = archive_input(bytes).await;
+ let reader = FullTextArchiveReader::from_input_file(&input)
+ .await
+ .unwrap();
+
+ // All three match, but restrict to row-ids {0, 2}.
+ let mut include = roaring::RoaringTreemap::new();
+ include.insert(0);
+ include.insert(2);
+
+ let hits = reader
+ .search_with_include(r#"{"match":{"query":"token"}}"#, 10,
&include)
+ .unwrap();
+ let mut ids = hits.row_ids.clone();
+ ids.sort_unstable();
+ assert_eq!(ids, vec![0, 2]);
+ }
+
+ #[tokio::test]
+ async fn test_search_with_include_empty_or_disjoint_returns_no_hits() {
+ let bytes = build_archive(&[
+ (0, "shared token here"),
+ (1, "shared token here"),
+ (2, "shared token here"),
+ ]);
+ let input = archive_input(bytes).await;
+ let reader = FullTextArchiveReader::from_input_file(&input)
+ .await
+ .unwrap();
+
+ // Empty allow-list: an empty include-set admits no rows (not all
rows).
+ let empty = roaring::RoaringTreemap::new();
+ let hits = reader
+ .search_with_include(r#"{"match":{"query":"token"}}"#, 10, &empty)
+ .unwrap();
+ assert!(
+ hits.row_ids.is_empty(),
+ "empty include-set must admit no rows"
+ );
+
+ // Allow-list disjoint from the matches (row 99 never indexed): still
nothing.
+ let mut disjoint = roaring::RoaringTreemap::new();
+ disjoint.insert(99);
+ let hits = reader
+ .search_with_include(r#"{"match":{"query":"token"}}"#, 10,
&disjoint)
+ .unwrap();
+ assert!(
+ hits.row_ids.is_empty(),
+ "include-set disjoint from matches must admit no rows"
+ );
+ }
+
+ #[test]
+ fn test_from_seek_read_opens_archive_directly() {
+ // Exercise the generic constructor directly (the key API the remote
+ // FileIO path in #571 depends on), bypassing `from_input_file`.
+ let bytes = build_archive(&[(0, "alpha bravo"), (1, "bravo charlie")]);
+ let reader =
+
FullTextArchiveReader::from_seek_read(BytesReader::new(Bytes::from(bytes))).unwrap();
+
+ let hits = reader.search(r#"{"match":{"query":"bravo"}}"#,
10).unwrap();
+ let mut ids = hits.row_ids.clone();
+ ids.sort_unstable();
+ assert_eq!(ids, vec![0, 1]);
+ }
+}
diff --git a/crates/paimon/src/lib.rs b/crates/paimon/src/lib.rs
index ddf207d3..85ad6b51 100644
--- a/crates/paimon/src/lib.rs
+++ b/crates/paimon/src/lib.rs
@@ -30,6 +30,8 @@ pub mod btree;
pub mod catalog;
mod deletion_vector;
pub mod file_index;
+#[cfg(feature = "fulltext")]
+pub mod ftindex;
pub mod io;
pub mod lumina;
mod predicate_stats;
