This is an automated email from the ASF dual-hosted git repository.
discivigour pushed a change to branch feat/blobPara
in repository https://gitbox.apache.org/repos/asf/paimon.git
omit 6adb20463c test(python): cover parallel blob fallback end to end
omit 2147be2f40 fix
omit ebac66c717 feat(python): parallelize fallback blob reads
add b524bae13a [core] Add LSM maintenance for sorted primary-key indexes
(#8613)
add 711481bea2 [core][connectors][docs] Complete primary-key sorted-index
support (#8615)
add 18fcff65e3 [python] Close exhausted blob fallback readers to improve
performance (#8622)
add 0a084375cf [flink] Fix incorrect argument names in action help
messages (#8612)
add 34bd5a16e8 [flink] Fix migrate actions crashing when parallelism is
not specified (#8611)
add c2e3a76196 [flink] Fix scan.partitions error message placeholder in
PartitionLoader (#8608)
add bafcd97065 [filesystem] Correct OBSLoader Javadoc to Huawei OBS
Storage (#8593)
add 8ba25a80bd [e2e] Fix mislabeled result sink column in
FileStoreBatchE2eTest (#8592)
add 2f36427b07 [common] Fix bounded bulk reads past end of stream (#8590)
add 8852d82fc1 [obs] FileSystem should be cached by authority (#8589)
add 298f031da8 [filesystem] Fix S3Loader Javadoc referring to oss instead
of s3 (#8587)
add 045d8f0e2c [core] Limit vector live-row planning to indexed ranges
(#8586)
add 67779ac54e [python] Introduce 'listagg' aggregator (#8561)
add 9a5661e5f1 [format] Support narrowing Parquet reads for TINYINT and
SMALLINT (#8629)
add 6211b65151 [format] Fix isNaN filter push-down failure on ORC tables
(#8627)
add 558765aa0a [flink] Support FLIP-510 key-only deletes on primary-key
table sink (#8452)
add db2c6183f0 [core][flink][spark] Support managed BLOBs in primary-key
tables (#8617)
add dd2a065b64 [feat] vector index: upgrade lumina-jni to 0.4.0 (#8621)
add 855de94e3b [core] Fix primary-key global-index scan planning (#8647)
add 31a3e0a293 [core] Add physical search and full-text archive primitives
(#8649)
add 803ed5c8ce [core] Maintain primary-key full-text index archives (#8651)
add c335a95049 [fix] Fix doc compile issue (#8656)
add 16af7fd137 [core] Search primary-key full-text indexes (#8652)
add bbf4045ba6 [hive] Correct column.name.delimiter property key in
HiveSchema (#8641)
add 9a302e7d08 [python][ray] Fix zero-block overwrite (#8632)
add 83cd9965a2 [core] Harden managed BLOB writes in primary-key tables
(#8654)
add f7ec93b3cc [flink] SortedIndexTopoBuilder supports custom commit-user
(#8646)
add 3e0c4e6bc7 pufeat(python): parallelize fallback blob reads
add c0883180ce fix
add 3572fd8596 test(python): cover parallel blob fallback end to end
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (6adb20463c)
\
N -- N -- N refs/heads/feat/blobPara (3572fd8596)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
docs/docs/primary-key-table/blob-storage.md | 137 +++++
docs/docs/primary-key-table/global-index.mdx | 380 ++++++++++++++
docs/docs/primary-key-table/vector-index.md | 255 ----------
docs/generated/core_configuration.html | 18 +-
docs/generated/flink_connector_configuration.html | 6 +
docs/redirects.js | 8 +
docs/sidebars.js | 3 +-
.../main/java/org/apache/paimon/CoreOptions.java | 88 +++-
.../apache/paimon/data/columnar/ColumnarArray.java | 14 +-
.../apache/paimon/data/columnar/ColumnarRow.java | 6 +-
.../data/serializer/InternalArraySerializer.java | 14 +
.../paimon/fs/OffsetSeekableInputStream.java | 13 +-
.../bitmap/BitmapGlobalIndexFormat.java | 130 +++--
.../bitmap/BitmapGlobalIndexWriter.java | 87 +++-
.../globalindex/bitmap/BitmapIndexReader.java | 37 +-
.../serializer/InternalArraySerializerTest.java | 34 ++
.../apache/paimon/format/FormatReadWriteTest.java | 39 ++
.../paimon/fs/OffsetSeekableInputStreamTest.java | 51 ++
.../bitmap/LazyFilteredBitmapIndexReaderTest.java | 141 +++++-
.../java/org/apache/paimon/KeyValueFileStore.java | 1 +
.../paimon/blob/ManagedBlobReferenceCollector.java | 133 +++++
.../paimon/blob/ManagedBlobReferenceFile.java | 196 ++++++++
.../paimon/blob/PrimaryKeyBlobExternalizer.java | 301 +++++++++++
.../sorted/SortedGlobalIndexBuilder.java | 24 +-
.../sorted/SortedSingleColumnIndexWriter.java | 20 +-
.../org/apache/paimon/index/IndexFileHandler.java | 5 +
.../pk/BucketedPrimaryKeyIndexMaintainer.java | 179 ++++++-
.../paimon/index/pk/PrimaryKeyIndexDefinition.java | 23 +-
.../index/pk/PrimaryKeyIndexDefinitions.java | 32 +-
.../PrimaryKeyIndexLevels.java} | 87 ++--
.../BucketedFullTextIndexMaintainer.java | 557 +++++++++++++++++++++
.../pkfulltext/PkFullTextBucketIndexState.java | 113 +++++
.../PkFullTextDataFileReader.java} | 58 +--
.../index/pkfulltext/PkFullTextIndexBuilder.java | 70 +++
.../index/pkfulltext/PkFullTextIndexFile.java | 233 +++++++++
.../pkfulltext/PrimaryKeyFullTextBucketSearch.java | 262 ++++++++++
.../pksorted/BucketedSortedIndexMaintainer.java | 270 +++++-----
.../index/pksorted/PkSortedBucketIndexState.java | 52 +-
.../index/pksorted/PkSortedIndexBuilder.java | 62 ++-
.../paimon/index/pksorted/PkSortedIndexFile.java | 118 +++--
.../paimon/index/pksorted/PkSortedIndexGroup.java | 74 ++-
.../pkvector/BucketedVectorIndexMaintainer.java | 18 +-
.../apache/paimon/io/KeyValueDataFileWriter.java | 68 ++-
.../paimon/io/KeyValueDataFileWriterImpl.java | 7 +-
.../paimon/io/KeyValueFileWriterFactory.java | 55 +-
.../paimon/io/KeyValueThinDataFileWriterImpl.java | 7 +-
.../apache/paimon/mergetree/MergeTreeWriter.java | 7 +-
.../paimon/operation/LocalOrphanFilesClean.java | 1 +
.../apache/paimon/operation/OrphanFilesClean.java | 9 +
.../operation/PrimaryKeyIndexedSplitRead.java | 4 +-
.../paimon/postpone/PostponeBucketWriter.java | 34 +-
.../org/apache/paimon/schema/SchemaManager.java | 3 +-
.../org/apache/paimon/schema/SchemaValidation.java | 145 +++++-
.../paimon/table/source/AbstractVectorRead.java | 11 +-
...eadImpl.java => DataEvolutionFullTextRead.java} | 9 +-
...canImpl.java => DataEvolutionFullTextScan.java} | 12 +-
.../table/source/FullTextSearchBuilderImpl.java | 43 +-
.../table/source/GlobalIndexLiveRowFilter.java | 22 +-
.../table/source/HybridSearchBuilderImpl.java | 5 +
.../paimon/table/source/PrimaryKeyBatchScan.java | 79 +--
.../table/source/PrimaryKeyFullTextRead.java | 224 +++++++++
...VectorScan.java => PrimaryKeyFullTextScan.java} | 214 ++++----
...lit.java => PrimaryKeyFullTextSearchSplit.java} | 100 ++--
...der.java => PrimaryKeyIndexPositionReader.java} | 6 +-
...ctorResult.java => PrimaryKeyScoredResult.java} | 187 +++----
.../table/source/PrimaryKeySearchPosition.java | 139 +++++
.../table/source/PrimaryKeySearchRanker.java | 212 ++++++++
.../table/source/PrimaryKeySortedIndexScan.java | 537 ++++++++++++++++++--
.../source/PrimaryKeyVectorPositionReader.java | 76 +--
.../table/source/PrimaryKeyVectorResult.java | 197 ++------
.../paimon/table/source/PrimaryKeyVectorScan.java | 41 +-
.../table/source/VectorSearchBuilderImpl.java | 12 +-
.../blob/ManagedBlobReferenceCollectorTest.java | 158 ++++++
.../paimon/blob/ManagedBlobReferenceFileTest.java | 111 ++++
.../blob/PrimaryKeyBlobExternalizerTest.java | 383 ++++++++++++++
.../sorted/SortedGlobalIndexBuilderTest.java | 51 ++
.../pk/BucketedPrimaryKeyIndexMaintainerTest.java | 223 ++++++++-
.../index/pk/PrimaryKeyIndexDefinitionsTest.java | 67 +++
.../paimon/index/pk/PrimaryKeyIndexLevelsTest.java | 171 +++++++
.../BucketedFullTextIndexMaintainerTest.java | 343 +++++++++++++
.../pkfulltext/PkFullTextBucketIndexStateTest.java | 102 ++++
.../PkFullTextDataFileReaderTest.java} | 54 +-
.../index/pkfulltext/PkFullTextIndexFileTest.java | 278 ++++++++++
.../PrimaryKeyFullTextBucketSearchTest.java | 416 +++++++++++++++
.../BucketedSortedIndexMaintainerTest.java | 433 +++++++++++++---
.../pksorted/PkSortedBucketIndexStateTest.java | 129 ++++-
.../index/pksorted/PkSortedIndexBuilderTest.java | 107 +++-
.../index/pksorted/PkSortedIndexFileTest.java | 190 +++++--
.../PrimaryKeySortedIndexMaintenanceTest.java | 24 +-
.../pksorted/PrimaryKeySortedIndexOptionsTest.java | 25 +-
.../BucketedVectorIndexMaintainerTest.java | 6 +-
.../index/pkvector/PkVectorAnnLevelsTest.java | 141 ------
.../paimon/io/PrimaryKeyBlobFileWriterTest.java | 148 ++++++
.../operation/LocalOrphanFilesCleanTest.java | 24 +
.../paimon/operation/PrimaryKeyIndexWriteTest.java | 34 ++
.../operation/PrimaryKeyIndexedSplitReadTest.java | 4 +-
.../operation/PrimaryKeyManagedBlobStoreTest.java | 414 +++++++++++++++
.../paimon/postpone/PostponeBucketWriterTest.java | 80 +++
... => PrimaryKeyFullTextIndexValidationTest.java} | 222 ++++----
.../PrimaryKeySortedIndexValidationTest.java | 11 +
.../PrimaryKeyVectorIndexValidationTest.java | 8 +-
.../apache/paimon/schema/SchemaManagerTest.java | 36 ++
.../apache/paimon/schema/SchemaValidationTest.java | 167 ++++++
.../paimon/table/BitmapGlobalIndexTableTest.java | 19 +-
.../table/source/FullTextSearchBuilderTest.java | 83 ++-
.../table/source/PrimaryKeyFullTextReadTest.java | 162 ++++++
.../table/source/PrimaryKeyFullTextScanTest.java | 307 ++++++++++++
.../table/source/PrimaryKeyFullTextSearchTest.java | 120 +++++
.../table/source/PrimaryKeyScoredResultTest.java | 96 ++++
.../table/source/PrimaryKeySearchRankerTest.java | 103 ++++
.../source/PrimaryKeySortedIndexBatchScanTest.java | 61 ++-
.../source/PrimaryKeySortedIndexReadTest.java | 61 ++-
.../source/PrimaryKeySortedIndexResultTest.java | 12 +-
.../source/PrimaryKeySortedIndexScanTest.java | 151 +++++-
.../source/PrimaryKeyVectorPositionReaderTest.java | 3 +-
.../table/source/PrimaryKeyVectorResultTest.java | 6 +-
.../table/source/PrimaryKeyVectorScanTest.java | 11 +-
.../table/source/VectorSearchBuilderTest.java | 50 ++
.../apache/paimon/tests/FileStoreBatchE2eTest.java | 2 +-
.../apache/paimon/obs/HadoopCompliantFileIO.java | 22 +-
.../main/java/org/apache/paimon/obs/OBSLoader.java | 2 +-
.../main/java/org/apache/paimon/s3/S3Loader.java | 2 +-
.../java/org/apache/paimon/flink/FlinkCatalog.java | 2 +-
.../apache/paimon/flink/FlinkConnectorOptions.java | 14 +
.../flink/action/CreateTagActionFactory.java | 3 +-
.../CreateTagFromTimestampActionFactory.java | 2 +-
.../CreateTagFromWatermarkActionFactory.java | 2 +-
.../action/ExpireChangelogsActionFactory.java | 2 +-
.../action/ExpirePartitionsActionFactory.java | 1 -
.../flink/action/ExpireSnapshotsActionFactory.java | 2 +-
.../flink/action/MigrateDatabaseActionFactory.java | 6 +-
.../flink/action/MigrateTableActionFactory.java | 6 +-
.../flink/action/RenameTagActionFactory.java | 2 +
.../action/RollbackToTimestampActionFactory.java | 2 +-
.../flink/globalindex/SortedIndexTopoBuilder.java | 22 +-
.../paimon/flink/lookup/PartitionLoader.java | 2 +-
.../paimon/flink/orphan/FlinkOrphanFilesClean.java | 4 +-
.../paimon/flink/sink/FlinkTableSinkBase.java | 25 +
.../org/apache/paimon/flink/BlobTableITCase.java | 40 ++
.../org/apache/paimon/flink/ChangelogModeTest.java | 25 +
.../apache/paimon/flink/KeyOnlyDeletesITCase.java | 183 +++++++
.../paimon/flink/PrimaryKeySortedIndexITCase.java | 112 +++++
.../action/MigrateDatabaseActionFactoryTest.java | 67 +++
.../action/MigrateTableActionFactoryTest.java | 67 +++
.../globalindex/SortedIndexTopoBuilderTest.java | 56 +++
.../lookup/DynamicPartitionLevelLoaderTest.java | 16 +
.../paimon/flink/utils/ChangelogModeUtils.java} | 15 +-
.../paimon/flink/utils/ChangelogModeUtils.java | 56 +++
.../orc/filter/OrcPredicateFunctionVisitor.java | 7 +
.../reader/ParquetVectorUpdaterFactory.java | 86 +++-
.../parquet/writer/ParquetRowDataWriter.java | 3 +-
.../format/avro/AvroFormatReadWriteTest.java | 7 +
.../paimon/format/orc/OrcFormatReadWriteTest.java | 5 +
.../format/orc/filter/OrcFilterConverterTest.java | 17 +
.../format/parquet/ParquetFormatReadWriteTest.java | 5 +
.../reader/FileTypeNotMatchReadTypeTest.java | 78 +++
.../index/NativePrimaryKeyFullTextIndexTest.java | 267 ++++++++++
.../java/org/apache/paimon/hive/HiveSchema.java | 2 +-
.../apache/paimon/hive/HiveTableSchemaTest.java | 23 +
paimon-lumina/pom.xml | 2 +-
.../pypaimon/common/options/core_options.py | 23 +
.../pypaimon/read/reader/aggregate/aggregators.py | 85 ++++
.../pypaimon/read/reader/concat_batch_reader.py | 15 +-
paimon-python/pypaimon/tests/blob_test.py | 94 ++++
.../pypaimon/tests/ray_integration_test.py | 25 +
paimon-python/pypaimon/tests/ray_sink_test.py | 35 ++
.../pypaimon/tests/test_field_aggregators.py | 302 ++++++++++-
paimon-python/pypaimon/write/ray_datasink.py | 4 +
.../spark/procedure/SparkOrphanFilesClean.scala | 1 +
.../DefaultGlobalIndexTopoBuilderTest.java | 7 +
.../org/apache/paimon/spark/sql/BlobTestBase.scala | 20 +
.../spark/sql/PrimaryKeySortedIndexTest.scala | 128 +++++
172 files changed, 12703 insertions(+), 1920 deletions(-)
create mode 100644 docs/docs/primary-key-table/blob-storage.md
create mode 100644 docs/docs/primary-key-table/global-index.mdx
delete mode 100644 docs/docs/primary-key-table/vector-index.md
create mode 100644
paimon-core/src/main/java/org/apache/paimon/blob/ManagedBlobReferenceCollector.java
create mode 100644
paimon-core/src/main/java/org/apache/paimon/blob/ManagedBlobReferenceFile.java
create mode 100644
paimon-core/src/main/java/org/apache/paimon/blob/PrimaryKeyBlobExternalizer.java
rename
paimon-core/src/main/java/org/apache/paimon/index/{pkvector/PkVectorAnnLevels.java
=> pk/PrimaryKeyIndexLevels.java} (61%)
create mode 100644
paimon-core/src/main/java/org/apache/paimon/index/pkfulltext/BucketedFullTextIndexMaintainer.java
create mode 100644
paimon-core/src/main/java/org/apache/paimon/index/pkfulltext/PkFullTextBucketIndexState.java
copy
paimon-core/src/main/java/org/apache/paimon/index/{pkvector/PkVectorDataFileReader.java
=> pkfulltext/PkFullTextDataFileReader.java} (62%)
create mode 100644
paimon-core/src/main/java/org/apache/paimon/index/pkfulltext/PkFullTextIndexBuilder.java
create mode 100644
paimon-core/src/main/java/org/apache/paimon/index/pkfulltext/PkFullTextIndexFile.java
create mode 100644
paimon-core/src/main/java/org/apache/paimon/index/pkfulltext/PrimaryKeyFullTextBucketSearch.java
rename
paimon-core/src/main/java/org/apache/paimon/table/source/{FullTextReadImpl.java
=> DataEvolutionFullTextRead.java} (97%)
rename
paimon-core/src/main/java/org/apache/paimon/table/source/{FullTextScanImpl.java
=> DataEvolutionFullTextScan.java} (97%)
create mode 100644
paimon-core/src/main/java/org/apache/paimon/table/source/PrimaryKeyFullTextRead.java
copy
paimon-core/src/main/java/org/apache/paimon/table/source/{PrimaryKeyVectorScan.java
=> PrimaryKeyFullTextScan.java} (57%)
copy
paimon-core/src/main/java/org/apache/paimon/table/source/{BucketVectorSearchSplit.java
=> PrimaryKeyFullTextSearchSplit.java} (50%)
copy
paimon-core/src/main/java/org/apache/paimon/table/source/{PrimaryKeyVectorPositionReader.java
=> PrimaryKeyIndexPositionReader.java} (94%)
copy
paimon-core/src/main/java/org/apache/paimon/table/source/{PrimaryKeyVectorResult.java
=> PrimaryKeyScoredResult.java} (51%)
create mode 100644
paimon-core/src/main/java/org/apache/paimon/table/source/PrimaryKeySearchPosition.java
create mode 100644
paimon-core/src/main/java/org/apache/paimon/table/source/PrimaryKeySearchRanker.java
create mode 100644
paimon-core/src/test/java/org/apache/paimon/blob/ManagedBlobReferenceCollectorTest.java
create mode 100644
paimon-core/src/test/java/org/apache/paimon/blob/ManagedBlobReferenceFileTest.java
create mode 100644
paimon-core/src/test/java/org/apache/paimon/blob/PrimaryKeyBlobExternalizerTest.java
create mode 100644
paimon-core/src/test/java/org/apache/paimon/index/pk/PrimaryKeyIndexLevelsTest.java
create mode 100644
paimon-core/src/test/java/org/apache/paimon/index/pkfulltext/BucketedFullTextIndexMaintainerTest.java
create mode 100644
paimon-core/src/test/java/org/apache/paimon/index/pkfulltext/PkFullTextBucketIndexStateTest.java
copy
paimon-core/src/test/java/org/apache/paimon/index/{pksorted/PkSortedDataFileReaderTest.java
=> pkfulltext/PkFullTextDataFileReaderTest.java} (77%)
create mode 100644
paimon-core/src/test/java/org/apache/paimon/index/pkfulltext/PkFullTextIndexFileTest.java
create mode 100644
paimon-core/src/test/java/org/apache/paimon/index/pkfulltext/PrimaryKeyFullTextBucketSearchTest.java
delete mode 100644
paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorAnnLevelsTest.java
create mode 100644
paimon-core/src/test/java/org/apache/paimon/io/PrimaryKeyBlobFileWriterTest.java
create mode 100644
paimon-core/src/test/java/org/apache/paimon/operation/PrimaryKeyManagedBlobStoreTest.java
create mode 100644
paimon-core/src/test/java/org/apache/paimon/postpone/PostponeBucketWriterTest.java
copy
paimon-core/src/test/java/org/apache/paimon/schema/{PrimaryKeyVectorIndexValidationTest.java
=> PrimaryKeyFullTextIndexValidationTest.java} (51%)
create mode 100644
paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeyFullTextReadTest.java
create mode 100644
paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeyFullTextScanTest.java
create mode 100644
paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeyFullTextSearchTest.java
create mode 100644
paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeyScoredResultTest.java
create mode 100644
paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeySearchRankerTest.java
create mode 100644
paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/KeyOnlyDeletesITCase.java
create mode 100644
paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/PrimaryKeySortedIndexITCase.java
create mode 100644
paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/MigrateDatabaseActionFactoryTest.java
create mode 100644
paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/MigrateTableActionFactoryTest.java
copy
paimon-flink/{paimon-flink-common/src/main/java/org/apache/paimon/flink/utils/BoundedOneInputOperator.java
=>
paimon-flink1-common/src/main/java/org/apache/paimon/flink/utils/ChangelogModeUtils.java}
(65%)
create mode 100644
paimon-flink/paimon-flink2-common/src/main/java/org/apache/paimon/flink/utils/ChangelogModeUtils.java
create mode 100644
paimon-full-text/src/test/java/org/apache/paimon/fulltext/index/NativePrimaryKeyFullTextIndexTest.java
create mode 100644
paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/PrimaryKeySortedIndexTest.scala
