This is an automated email from the ASF dual-hosted git repository.
leaves12138 pushed a change to branch native-reassign
in repository https://gitbox.apache.org/repos/asf/paimon.git
discard 82b9cf4f47 Update reassign for java
add 9956a067fc [python] optimize blob bunch construction (#8859)
add e818190e49 [core] Reject non-positive global index read thread counts
(#8853)
add 5652c0023c [test] Fix unstable unaware bucket read test and assert
split distribution (#8850)
add 095a1d2ea9 [core] Improve LocalKvDb lookup and compaction performance
(#8857)
add 881e924398 [core] Support off-heap lookup cache (#8867)
add 6695d6c6f5 [core] Optimize ZSTD block decompression (#8865)
add dbc0c024bc [core] Extract lookup state bulk sorter utility (#8868)
add 2e8f69a154 [core] Make LocalKvDb flush failure-safe (#8869)
add 59003b520f [core] Support streaming LocalKvDb bulk load (#8870)
add 5f6876c025 [flink] Support distributed vector search (#8834)
add 5c45919aef [core] Add version-aware LocalKvDb range iterator (#8871)
add e599991d83 [core] Support LocalKvDb merge and TTL compaction (#8872)
add 0e3f4461cc [hive] Populate partition done status from metastore events
(#8866)
add 81c935edad [core] Add LocalKvDb-backed lookup states (#8880)
add 9ad20d9978 [spark] Infer postpone bucket count from data size (#8864)
new 17636a9d23 [core] Reduce row-id reassignment planning memory
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (82b9cf4f47)
\
N -- N -- N refs/heads/native-reassign (17636a9d23)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
docs/docs/flink/procedures.md | 12 +-
docs/docs/multimodal-table/global-index/vector.mdx | 4 +-
docs/docs/primary-key-table/data-distribution.md | 25 +-
docs/docs/primary-key-table/global-index.mdx | 5 +-
docs/generated/core_configuration.html | 16 +-
.../main/java/org/apache/paimon/CoreOptions.java | 34 +-
paimon-benchmark/paimon-micro-benchmarks/pom.xml | 7 +
.../org/apache/paimon/benchmark/Benchmark.java | 8 +-
.../benchmark/cache/CacheManagerBenchmark.java | 49 +-
.../benchmark/lookup/AbstractLookupBenchmark.java | 9 +-
.../benchmark/lookup/LocalKvDbBenchmark.java | 649 ++++++++++
.../paimon/compression/ZstdBlockDecompressor.java | 38 +-
.../java/org/apache/paimon/io/cache/Cache.java | 8 +-
.../org/apache/paimon/io/cache/CacheBuilder.java | 68 +-
.../java/org/apache/paimon/io/cache/CacheKey.java | 16 +-
.../org/apache/paimon/io/cache/CacheManager.java | 48 +-
.../org/apache/paimon/io/cache/GuavaCache.java | 68 -
.../apache/paimon/lookup/LookupStoreFactory.java | 12 +-
.../paimon/lookup/sort/SortLookupStoreFactory.java | 6 +-
.../paimon/lookup/sort/SortLookupStoreReader.java | 9 +
.../paimon/lookup/sort/SortLookupStoreWriter.java | 4 +-
.../paimon/lookup/sort/db/AsyncLsmCompactor.java | 114 ++
.../apache/paimon/lookup/sort/db/LocalKvDb.java | 1342 ++++++++++++++++++++
.../apache/paimon/lookup/sort/db/LsmCompactor.java | 746 +----------
.../apache/paimon/lookup/sort/db/LsmLevels.java | 374 ++++++
.../lookup/sort/db/RecordCombiningWriter.java | 111 ++
.../paimon/lookup/sort/db/SimpleLsmKvDb.java | 693 ----------
.../paimon/lookup/sort/db/SyncLsmCompactor.java | 33 +-
.../{LsmCompactor.java => UniversalCompactor.java} | 275 ++--
.../java/org/apache/paimon/sst/BlockIterator.java | 30 +-
.../java/org/apache/paimon/sst/SstFileWriter.java | 20 +-
.../main/java/org/apache/paimon/utils/BitSet.java | 7 +
.../java/org/apache/paimon/utils/BloomFilter.java | 76 +-
.../paimon/utils/DynamicBloomFilterBuilder.java | 76 ++
.../paimon/utils/FixedBloomFilterBuilder.java | 58 +
.../globalindex/btree/AbstractIndexReaderTest.java | 2 +-
.../globalindex/btree/BTreeThreadSafetyTest.java | 2 +-
.../apache/paimon/io/cache/CacheManagerTest.java | 50 +-
.../lookup/sort/SortLookupStoreFactoryTest.java | 16 +-
.../paimon/lookup/sort/SortLookupStoreTest.java | 12 +-
.../sort/db/LocalKvDbAsyncCompactionTest.java | 329 +++++
.../{SimpleLsmKvDbTest.java => LocalKvDbTest.java} | 823 ++++++++++--
.../org/apache/paimon/sst/BlockIteratorTest.java | 8 +
.../org/apache/paimon/utils/BloomFilterTest.java | 59 +-
.../paimon/utils/FileBasedBloomFilterTest.java | 27 +-
.../globalindex/GlobalIndexReadThreadPool.java | 2 +
.../StateUtils.java} | 36 +-
.../paimon/lookup/local/LocalKvBulkLoader.java | 63 +
.../paimon/lookup/local/LocalKvCompositeKey.java | 86 ++
.../paimon/lookup/local/LocalKvListBulkLoader.java | 87 ++
.../lookup/local/LocalKvListMergeOperator.java | 68 +
.../paimon/lookup/local/LocalKvListState.java | 120 ++
.../paimon/lookup/local/LocalKvListValueCodec.java | 187 +++
.../paimon/lookup/local/LocalKvSetState.java | 109 ++
.../apache/paimon/lookup/local/LocalKvState.java | 117 ++
.../paimon/lookup/local/LocalKvStateFactory.java | 201 +++
.../paimon/lookup/local/LocalKvValueCodec.java | 127 ++
.../paimon/lookup/local/LocalKvValueState.java | 95 ++
.../apache/paimon/lookup/rocksdb/RocksDBState.java | 15 +-
.../org/apache/paimon/mergetree/LookupLevels.java | 8 +-
.../compact/MergeTreeCompactManagerFactory.java | 4 +-
.../clustering/ClusteringCompactManager.java | 8 +-
.../ClusteringCompactManagerFactory.java | 1 +
.../compact/clustering/ClusteringKeyIndex.java | 12 +-
.../apache/paimon/table/query/LocalTableQuery.java | 4 +-
.../table/source/VectorSearchBuilderImpl.java | 2 +-
.../java/org/apache/paimon/CoreOptionsTest.java | 19 +
.../globalindex/GlobalIndexReadThreadPoolTest.java | 23 +-
.../lookup/local/LocalKvStateFactoryTest.java | 562 ++++++++
.../paimon/mergetree/ContainsLevelsTest.java | 4 +-
.../apache/paimon/mergetree/LookupLevelsTest.java | 16 +-
.../paimon/separated/ClusteringTableTest.java | 27 +
.../flink/procedure/VectorSearchProcedure.java | 429 ++++++-
.../vectorsearch/FlinkDataEvolutionVectorRead.java | 590 +++++++++
.../vectorsearch/FlinkPrimaryKeyVectorRead.java | 121 +-
.../vectorsearch/FlinkVectorSearchBuilderImpl.java | 29 +-
.../apache/paimon/flink/AppendOnlyTableITCase.java | 16 -
.../procedure/VectorSearchProcedureITCase.java | 674 +++++++++-
.../source/UnawareBucketSplitShuffleITCase.java | 147 +++
.../FlinkDataEvolutionVectorReadTest.java | 555 ++++++++
.../FlinkPrimaryKeyVectorReadTest.java | 163 +++
.../java/org/apache/paimon/hive/HiveCatalog.java | 132 +-
.../org/apache/paimon/hive/HiveCatalogTest.java | 47 +
paimon-python/pypaimon/read/reader/field_bunch.py | 28 +-
paimon-python/pypaimon/read/split_read.py | 4 +-
paimon-python/pypaimon/tests/field_bunch_test.py | 130 ++
.../paimon/spark/commands/PaimonSparkWriter.scala | 196 ++-
.../spark/commands/WriteIntoPaimonTable.scala | 5 +-
.../paimon/spark/sql/PostponeBucketTableTest.scala | 219 +++-
89 files changed, 9685 insertions(+), 2181 deletions(-)
create mode 100644
paimon-benchmark/paimon-micro-benchmarks/src/test/java/org/apache/paimon/benchmark/lookup/LocalKvDbBenchmark.java
delete mode 100644
paimon-common/src/main/java/org/apache/paimon/io/cache/GuavaCache.java
create mode 100644
paimon-common/src/main/java/org/apache/paimon/lookup/sort/db/AsyncLsmCompactor.java
create mode 100644
paimon-common/src/main/java/org/apache/paimon/lookup/sort/db/LocalKvDb.java
create mode 100644
paimon-common/src/main/java/org/apache/paimon/lookup/sort/db/LsmLevels.java
create mode 100644
paimon-common/src/main/java/org/apache/paimon/lookup/sort/db/RecordCombiningWriter.java
delete mode 100644
paimon-common/src/main/java/org/apache/paimon/lookup/sort/db/SimpleLsmKvDb.java
copy
paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/NoOpEnumStateSerializer.java
=>
paimon-common/src/main/java/org/apache/paimon/lookup/sort/db/SyncLsmCompactor.java
(56%)
copy
paimon-common/src/main/java/org/apache/paimon/lookup/sort/db/{LsmCompactor.java
=> UniversalCompactor.java} (76%)
create mode 100644
paimon-common/src/main/java/org/apache/paimon/utils/DynamicBloomFilterBuilder.java
create mode 100644
paimon-common/src/main/java/org/apache/paimon/utils/FixedBloomFilterBuilder.java
create mode 100644
paimon-common/src/test/java/org/apache/paimon/lookup/sort/db/LocalKvDbAsyncCompactionTest.java
rename
paimon-common/src/test/java/org/apache/paimon/lookup/sort/db/{SimpleLsmKvDbTest.java
=> LocalKvDbTest.java} (64%)
copy
paimon-core/src/main/java/org/apache/paimon/{utils/IntObjectSerializer.java =>
lookup/StateUtils.java} (52%)
create mode 100644
paimon-core/src/main/java/org/apache/paimon/lookup/local/LocalKvBulkLoader.java
create mode 100644
paimon-core/src/main/java/org/apache/paimon/lookup/local/LocalKvCompositeKey.java
create mode 100644
paimon-core/src/main/java/org/apache/paimon/lookup/local/LocalKvListBulkLoader.java
create mode 100644
paimon-core/src/main/java/org/apache/paimon/lookup/local/LocalKvListMergeOperator.java
create mode 100644
paimon-core/src/main/java/org/apache/paimon/lookup/local/LocalKvListState.java
create mode 100644
paimon-core/src/main/java/org/apache/paimon/lookup/local/LocalKvListValueCodec.java
create mode 100644
paimon-core/src/main/java/org/apache/paimon/lookup/local/LocalKvSetState.java
create mode 100644
paimon-core/src/main/java/org/apache/paimon/lookup/local/LocalKvState.java
create mode 100644
paimon-core/src/main/java/org/apache/paimon/lookup/local/LocalKvStateFactory.java
create mode 100644
paimon-core/src/main/java/org/apache/paimon/lookup/local/LocalKvValueCodec.java
create mode 100644
paimon-core/src/main/java/org/apache/paimon/lookup/local/LocalKvValueState.java
copy
paimon-common/src/main/java/org/apache/paimon/globalindex/GlobalIndexSingleColumnWriter.java
=>
paimon-core/src/test/java/org/apache/paimon/globalindex/GlobalIndexReadThreadPoolTest.java
(57%)
create mode 100644
paimon-core/src/test/java/org/apache/paimon/lookup/local/LocalKvStateFactoryTest.java
create mode 100644
paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/vectorsearch/FlinkDataEvolutionVectorRead.java
copy
paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/read/SparkPrimaryKeyVectorRead.java
=>
paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/vectorsearch/FlinkPrimaryKeyVectorRead.java
(50%)
copy
paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/read/SparkVectorSearchBuilderImpl.java
=>
paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/vectorsearch/FlinkVectorSearchBuilderImpl.java
(63%)
create mode 100644
paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/UnawareBucketSplitShuffleITCase.java
create mode 100644
paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/vectorsearch/FlinkDataEvolutionVectorReadTest.java
create mode 100644
paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/vectorsearch/FlinkPrimaryKeyVectorReadTest.java
create mode 100644 paimon-python/pypaimon/tests/field_bunch_test.py
