This is an automated email from the ASF dual-hosted git repository.
discivigour pushed a change to branch feat/blobPara
in repository https://gitbox.apache.org/repos/asf/paimon.git
discard 295e97466b feat(python): parallelize fallback blob reads
add d41ec7357f [core] Read primary-key vector results by position (#8576)
add a50a36ff85 [core] Search primary-key vector indexes (#8579)
add 3d463ba392 [spark][flink] Support primary-key vector search (#8581)
add df71005c7c [iceberg] Fix UnsupportedOperationException reading
multiset columns (#8580)
add 960cb6a8e9 [filesystem] Fix misleading OBS comment in hadoop-uber
shade config (#8575)
add f218528437 [core] Clean up index files on aborted commits (#8572)
add 6509987c60 [avro] Optimize array in avro (#8567)
add 8393d28ba2 [python] Reject changing immutable options once a table has
snapshots (#8565)
add 3cfaa512cb [core] Mark table 'type' option as immutable (#8564)
add 73d38b6df3 [fs] Force CVE-patched commons-lang3 and snappy-java in
hadoop-shaded (#8560)
add 1b43de297a [fs] Fix GSFileIO to return actual file access time and
owner (#8559)
add ee61b54e32 [filesystem] Fix jersey-json typo in hadoop-shaded-3.4 pom
comment (#8557)
add 208764a0de [fs] Declare serialVersionUID in COSNLoader for
Serializable consistency (#8555)
add 1b821f595e [cosn] Log Hadoop config entries at debug level instead of
warn (#8554)
add aa41699e6f [docs] Escape ampersand (&) in config doc generator's
escapeCharacters (#8552)
add 2b2904d211 [python] Fix mock REST snapshot serialization (#8546)
add bb085744d7 [hive] Correct column.name.delimiter (#8545)
add f797982eb6 [spark] convert EqualTo(NaN) to isNaN in V2 filters (#8542)
add 450a993e02 [python][ray] Fix HASH_FIXED primary-key writer ownership
(#8541)
add 8490f4d5c9 [spark] Reject SET LOCATION for Paimon tables (#8540)
add fa62f98a7f [spark] Support V2 delta-based row-level operations for
deletion-vector append tables (#8539)
add ee50c951fe [common] Fix map equality for binary keys (#8536)
add 8b51ac5bc6 [codegen] Fix floating point record equaliser semantics
(#8534)
add 1379eefb21 [format] Support narrowing Parquet reads (#8505)
add d7e493341a [Blob] Add some metric when write blob from http. (#8464)
add 5cfb2cf659 [flink-cdc] Stabilize schema change IT under load (#8585)
add a7fa5594d3 [codegen-loader] Fix stale dependency comments (#8533)
add 23da742275 [core] Enforce LIMIT at the table-read level for
primary-key tables (#8116)
add 4a2d409f96 [core] Add asynchronous PK vector ANN compaction (#8584)
add ec1d8ee8ac [hotfix] Fix test error from Limit PR merging
add 7e396cff87 Add rerank to primary-key vector search (#8591)
add 26f415a1a2 [core][flink][spark] Support PK-vector indexes for postpone
buckets (#8594)
add f87fe0fb07 [core] Generalize source metadata for primary-key indexes
(#8595)
add 32e1032128 [docs] Add Variant storage documentation (#8597)
add 9107d0a6a1 [core] Generalize the primary-key index write lifecycle
(#8598)
add f31f199ebc [core] Define scalar primary-key global indexes (#8599)
add c8a87e9cbd [core] Build source-backed sorted index payloads (#8600)
add be80abff41 [core] Maintain sorted primary-key index payloads (#8601)
add 4dddb86aca [core] Wire sorted primary-key indexes into writes (#8602)
add 55e4358d8e [hotfix] Fix unstable test in
BucketedSortedIndexMaintainerTest
add cad041961c [core] add a `deleted_record_count` field in
PartitionsTable (#8526)
add 0376c8bb28 [flink][spark] Fix conflict error for no-op file index
rewrites (#8596)
add 4fe3cfdd13 [core] Prune primary-key scans and vector search with
sorted indexes (#8603)
add d37e503dc1 [global-index-eslib] integrates the Elasticsearch (Lucene)
index engine into Paimon's global index system (#8000)
add 1a44193ec2 [global-index] Fix full-text range selection and
scalar-filter coverage for multi-field indexes (#8547)
add e425fb42e8 [spark] Make DeletionVectorTest pass with
write.use-v2-write enabled (#8614)
add ee86522053 [core][flink][spark] Support ARRAY<BLOB> blob files (#8181)
add a910c615ac [global-index-eslib]use thin packaging and fix pom
bundle,fix query compatibility and vector metrics, and (#8618)
add ebac66c717 feat(python): parallelize fallback blob reads
add 2147be2f40 fix
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (295e97466b)
\
N -- N -- N refs/heads/feat/blobPara (2147be2f40)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
docs/docs/concepts/spec/fileformat.md | 22 +-
docs/docs/concepts/system-tables.mdx | 11 +-
docs/docs/maintenance/metrics.md | 86 +-
docs/docs/multimodal-table/blob.mdx | 78 +-
docs/docs/multimodal-table/index.mdx | 4 +-
docs/docs/multimodal-table/variant.mdx | 251 +++
docs/docs/primary-key-table/vector-index.md | 255 +++
docs/docs/pypaimon/ray-data.md | 34 +-
docs/docs/pypaimon/system-tables.md | 1 +
docs/generated/core_configuration.html | 24 +
docs/sidebars.js | 1 +
.../main/java/org/apache/paimon/CoreOptions.java | 101 +-
.../java/org/apache/paimon/types/BlobType.java | 13 +-
.../benchmark/compact/ArrayCompactBenchmark.java | 142 ++
paimon-codegen-loader/pom.xml | 4 +-
.../paimon/codegen/EqualiserCodeGenerator.scala | 20 +-
.../apache/paimon/codegen/ScalarOperatorGens.scala | 171 +-
.../paimon/codegen/EqualiserCodeGeneratorTest.java | 156 ++
.../apache/paimon/data/BlobArrayPlaceholder.java | 124 +-
.../paimon/data/BlobFetchMetricReporter.java | 42 +-
.../java/org/apache/paimon/data/InternalRow.java | 2 +
.../paimon/globalindex/GlobalIndexEvaluator.java | 69 +-
.../paimon/globalindex/UnionGlobalIndexReader.java | 9 +-
.../apache/paimon/reader/LimitRecordReader.java | 117 ++
...ileRecordReader.java => ScoreRecordReader.java} | 8 +-
.../org/apache/paimon/utils/InternalRowUtils.java | 49 +-
.../globalindex/GlobalIndexEvaluatorTest.java | 105 +-
.../TestFullTextGlobalIndexerFactory.java | 20 +
.../TestMultiFieldVectorGlobalIndexer.java | 261 +++
.../testvector/TestVectorGlobalIndexerFactory.java | 11 +
.../apache/paimon/utils/InternalRowUtilsTest.java | 54 +
.../java/org/apache/paimon/KeyValueFileStore.java | 24 +-
.../append/DedicatedFormatRollingFileWriter.java | 3 +-
.../paimon/append/MultipleBlobFileWriter.java | 5 +-
.../DataEvolutionCompactCoordinator.java | 4 +-
.../paimon/globalindex/DataEvolutionBatchScan.java | 6 +-
.../paimon/globalindex/GlobalIndexCoverage.java | 3 +
.../paimon/globalindex/GlobalIndexScanner.java | 67 +-
.../apache/paimon/globalindex/IndexedSplit.java | 8 +-
.../globalindex/IndexedSplitRecordReader.java | 3 +-
.../sorted/SortedGlobalIndexBuilder.java | 24 +-
.../sorted/SortedSingleColumnIndexWriter.java | 87 +
.../paimon/iceberg/metadata/IcebergDataField.java | 9 +
.../org/apache/paimon/index/IndexFileHandler.java | 5 +
.../pk/BucketedPrimaryKeyIndexMaintainer.java | 414 ++++
.../paimon/index/pk/PrimaryKeyIndexDefinition.java | 51 +-
.../index/pk/PrimaryKeyIndexDefinitions.java | 118 ++
.../PrimaryKeyIndexSourceFile.java} | 10 +-
.../PrimaryKeyIndexSourceMeta.java} | 59 +-
.../PrimaryKeyIndexSourcePolicy.java} | 8 +-
.../pksorted/BucketedSortedIndexMaintainer.java | 561 ++++++
.../index/pksorted/PkSortedBucketIndexState.java | 110 ++
.../PkSortedDataFileReader.java} | 98 +-
.../index/pksorted/PkSortedIndexBuilder.java | 179 ++
.../paimon/index/pksorted/PkSortedIndexFile.java | 202 ++
.../paimon/index/pksorted/PkSortedIndexGroup.java | 79 +
.../pkvector/BucketedVectorIndexMaintainer.java | 388 +++-
.../paimon/index/pkvector/PkVectorAnnLevels.java | 163 ++
.../index/pkvector/PkVectorAnnSegmentFile.java | 18 +-
.../index/pkvector/PkVectorAnnSegmentSearcher.java | 127 +-
.../index/pkvector/PkVectorBucketIndexState.java | 10 +-
.../pkvector/PrimaryKeyVectorBucketSearch.java | 168 +-
.../paimon/operation/AbstractFileStoreWrite.java | 100 +-
.../operation/AllPlaceholdersRecordReader.java | 9 +-
.../paimon/operation/BaseAppendFileStoreWrite.java | 21 +
.../paimon/operation/BlobFallbackRecordReader.java | 17 +-
.../apache/paimon/operation/BlobFileContext.java | 23 +-
.../paimon/operation/DataEvolutionSplitRead.java | 4 +-
.../paimon/operation/FileStoreCommitImpl.java | 34 +-
.../apache/paimon/operation/FileStoreWrite.java | 10 +-
.../paimon/operation/FileSystemWriteRestore.java | 37 +-
.../paimon/operation/KeyValueFileStoreWrite.java | 6 +-
.../paimon/operation/MemoryFileStoreWrite.java | 6 +-
.../operation/PrimaryKeyIndexedSplitRead.java | 121 ++
.../apache/paimon/operation/RawFileSplitRead.java | 88 +-
.../org/apache/paimon/operation/RestoreFiles.java | 10 +-
.../org/apache/paimon/operation/WriteRestore.java | 2 +-
.../paimon/operation/metrics/BlobFetchMetrics.java | 146 ++
.../org/apache/paimon/postpone/BucketFiles.java | 2 +
.../paimon/privilege/PrivilegedFileStoreTable.java | 6 +
.../apache/paimon/schema/ColumnDirectiveUtils.java | 43 +-
.../org/apache/paimon/schema/SchemaManager.java | 69 +-
.../org/apache/paimon/schema/SchemaValidation.java | 147 +-
.../paimon/table/AbstractFileStoreTable.java | 24 -
.../paimon/table/AppendOnlyFileStoreTable.java | 21 +
.../apache/paimon/table/ChainGroupReadTable.java | 35 +-
.../paimon/table/DelegatedFileStoreTable.java | 5 +
.../paimon/table/FallbackReadFileStoreTable.java | 9 +-
.../org/apache/paimon/table/FileStoreTable.java | 12 +
.../org/apache/paimon/table/PostponeUtils.java | 116 +-
.../paimon/table/PrimaryKeyFileStoreTable.java | 16 +
.../paimon/table/format/FormatTableRead.java | 48 +-
...eBatchScan.java => AbstractBatchTableScan.java} | 70 +-
.../paimon/table/source/AbstractDataTableScan.java | 43 +-
.../paimon/table/source/AbstractVectorRead.java | 84 +-
.../paimon/table/source/AppendBatchTableScan.java | 26 +-
.../table/source/BatchVectorSearchBuilderImpl.java | 2 +-
.../table/source/BucketVectorSearchSplit.java | 133 ++
...rReadImpl.java => DataEvolutionVectorRead.java} | 6 +-
...rScanImpl.java => DataEvolutionVectorScan.java} | 28 +-
.../paimon/table/source/FullTextReadImpl.java | 58 +-
.../paimon/table/source/FullTextScanImpl.java | 284 ++-
...llTextRead.java => GlobalIndexSplitResult.java} | 10 +-
.../table/source/IndexFullTextSearchSplit.java | 41 +-
.../paimon/table/source/KeyValueTableRead.java | 17 +-
.../paimon/table/source/PrimaryKeyBatchScan.java | 171 ++
.../table/source/PrimaryKeySortedIndexResult.java | 143 ++
.../table/source/PrimaryKeySortedIndexScan.java | 358 ++++
.../source/PrimaryKeyVectorPositionReader.java | 106 +
.../paimon/table/source/PrimaryKeyVectorRead.java | 640 ++++++
.../table/source/PrimaryKeyVectorResult.java | 228 +++
.../paimon/table/source/PrimaryKeyVectorScan.java | 299 +++
.../table/source/VectorSearchBuilderImpl.java | 20 +-
.../table/source/VectorSearchRefineOptions.java | 121 ++
.../table/source/snapshot/SnapshotReader.java | 4 +
.../table/source/snapshot/SnapshotReaderImpl.java | 5 +
...ava => PrimaryKeyIndexedSplitReadProvider.java} | 21 +-
.../apache/paimon/table/system/AuditLogTable.java | 35 +-
.../paimon/table/system/PartitionsTable.java | 74 +-
.../paimon/table/system/ReadOptimizedTable.java | 19 +-
.../test/java/org/apache/paimon/JavaPyE2ETest.java | 93 +
.../org/apache/paimon/append/BlobTableTest.java | 131 ++
.../globalindex/DataEvolutionBatchScanTest.java | 12 +-
.../sorted/SortedGlobalIndexBuilderTest.java | 36 +
.../iceberg/metadata/IcebergDataFieldTest.java | 22 +
.../pk/BucketedPrimaryKeyIndexMaintainerTest.java | 351 ++++
.../index/pk/PrimaryKeyIndexDefinitionsTest.java | 104 +
.../index/pk/PrimaryKeyIndexSourceMetaTest.java | 127 ++
.../BucketedSortedIndexMaintainerTest.java | 537 ++++++
.../pksorted/PkSortedBucketIndexStateTest.java | 206 ++
.../PkSortedDataFileReaderTest.java} | 59 +-
.../index/pksorted/PkSortedIndexBuilderTest.java | 294 +++
.../index/pksorted/PkSortedIndexFileTest.java | 186 ++
.../PrimaryKeySortedIndexMaintenanceTest.java | 396 ++++
.../pksorted/PrimaryKeySortedIndexOptionsTest.java | 78 +
.../BucketedVectorIndexMaintainerTest.java | 593 +++++-
.../index/pkvector/PkVectorAnnLevelsTest.java | 141 ++
.../index/pkvector/PkVectorAnnSegmentFileTest.java | 57 +-
.../pkvector/PkVectorBucketIndexStateTest.java | 6 +-
.../index/pkvector/PkVectorSourceMetaTest.java | 56 -
.../pkvector/PrimaryKeyVectorBucketSearchTest.java | 143 +-
.../operation/BlobFallbackRecordReaderTest.java | 70 +
.../paimon/operation/FileStoreCommitTest.java | 53 +
.../operation/FileSystemWriteRestoreTest.java | 43 +-
.../paimon/operation/PrimaryKeyIndexWriteTest.java | 197 ++
.../operation/PrimaryKeyIndexedSplitReadTest.java | 98 +
.../operation/PrimaryKeyVectorIndexWriteTest.java | 79 +-
.../operation/metrics/BlobFetchMetricsTest.java | 97 +
.../apache/paimon/postpone/BucketFilesTest.java | 72 +
.../paimon/schema/ColumnDirectiveUtilsTest.java | 63 +-
.../PrimaryKeySortedIndexValidationTest.java | 191 ++
.../PrimaryKeyVectorIndexValidationTest.java | 59 +-
.../apache/paimon/schema/SchemaManagerTest.java | 153 +-
.../apache/paimon/schema/SchemaValidationTest.java | 9 +-
.../paimon/table/BtreeGlobalIndexTableTest.java | 59 +
.../org/apache/paimon/table/PostponeUtilsTest.java | 106 +
.../paimon/table/PrimaryKeySimpleTableTest.java | 229 +++
.../apache/paimon/table/SchemaEvolutionTest.java | 3 +-
.../table/source/FullTextSearchBuilderTest.java | 225 ++-
.../source/PrimaryKeySortedIndexBatchScanTest.java | 243 +++
.../source/PrimaryKeySortedIndexReadTest.java | 149 ++
.../source/PrimaryKeySortedIndexResultTest.java | 287 +++
.../source/PrimaryKeySortedIndexScanTest.java | 368 ++++
.../source/PrimaryKeyVectorPositionReaderTest.java | 197 ++
.../PrimaryKeyVectorRawFileSplitReadTest.java | 108 ++
.../table/source/PrimaryKeyVectorReadTest.java | 58 +
.../table/source/PrimaryKeyVectorResultTest.java | 96 +
.../table/source/PrimaryKeyVectorScanTest.java | 289 +++
.../table/source/PrimaryKeyVectorSearchTest.java | 306 +++
.../apache/paimon/table/source/ScanBucketTest.java | 37 +-
.../apache/paimon/table/source/TableScanTest.java | 16 +
.../table/source/VectorSearchBuilderTest.java | 144 +-
.../source/VectorSearchRefineOptionsTest.java | 112 ++
.../paimon/table/system/AuditLogTableTest.java | 155 ++
.../paimon/table/system/PartitionsTableTest.java | 105 +
.../apache/paimon/tag/BatchReadTagCreatorTest.java | 11 +-
.../java/org/apache/paimon/docs/util/Utils.java | 1 +
.../org/apache/paimon/docs/util/UtilsTest.java | 32 +-
paimon-eslib/README.md | 221 +++
paimon-eslib/pom.xml | 124 ++
.../paimon/eslib/index/ESIndexBuilderFactory.java | 28 +-
.../apache/paimon/eslib/index/ESIndexFileMeta.java | 382 ++++
.../eslib/index/ESIndexGlobalIndexReader.java | 1953 +++++++++++++++++++
.../eslib/index/ESIndexGlobalIndexWriter.java | 629 ++++++
.../paimon/eslib/index/ESIndexGlobalIndexer.java | 111 ++
.../eslib/index/ESIndexGlobalIndexerFactory.java | 28 +-
.../apache/paimon/eslib/index/ESIndexOptions.java | 643 +++++++
....apache.paimon.globalindex.GlobalIndexerFactory | 2 +-
.../paimon/eslib/index/ESIndexFileMetaTest.java | 365 ++++
.../index/ESIndexFullTextQueryParserTest.java | 152 ++
.../eslib/index/ESIndexGlobalIndexE2ETest.java | 2034 ++++++++++++++++++++
.../index/ESIndexGlobalIndexReaderDebugTest.java | 82 +
.../index/ESIndexGlobalIndexerExecutorTest.java | 92 +
.../paimon/eslib/index/ESIndexOptionsTest.java | 403 ++++
.../eslib/index/ESIndexVectorMetricTest.java | 162 ++
.../java/org/apache/paimon/cosn/COSNFileIO.java | 2 +-
.../java/org/apache/paimon/cosn/COSNLoader.java | 3 +
.../apache/paimon/gs/HadoopCompliantFileIO.java | 10 +
.../paimon-hadoop-shaded-3.4/pom.xml | 2 +-
paimon-filesystems/paimon-hadoop-shaded/pom.xml | 12 +
paimon-filesystems/paimon-hadoop-uber/pom.xml | 2 +-
.../cdc/mysql/MySqlSyncDatabaseActionITCase.java | 5 +
.../paimon/flink/AbstractFlinkTableFactory.java | 30 +-
.../java/org/apache/paimon/flink/FlinkCatalog.java | 26 +-
.../apache/paimon/flink/FlinkRowDataWithBlob.java | 34 +-
.../org/apache/paimon/flink/FlinkRowWrapper.java | 42 +-
.../apache/paimon/flink/LogicalTypeConversion.java | 32 +-
.../apache/paimon/flink/action/CompactAction.java | 116 +-
.../flink/action/DataEvolutionMergeIntoAction.java | 5 +-
.../flink/lookup/FileStoreLookupFunction.java | 20 +-
.../paimon/flink/lookup/FullCacheLookupTable.java | 12 +-
.../postpone/PostponeBucketCompactOperator.java | 134 ++
.../postpone/PostponeBucketCompactSplitSource.java | 44 +-
.../paimon/flink/sink/RewriteFileIndexSink.java | 3 +
.../flink/source/FileStoreSourceSplitReader.java | 8 +-
.../org/apache/paimon/flink/BlobTableITCase.java | 124 ++
.../paimon/flink/FlinkRowDataWithBlobTest.java | 98 +
.../paimon/flink/KeyValueTableReadLimitITCase.java | 110 ++
.../org/apache/paimon/flink/LookupJoinITCase.java | 67 +
.../org/apache/paimon/flink/ScanBucketITCase.java | 48 -
.../action/DataEvolutionMergeIntoActionITCase.java | 84 +-
.../PostponeBucketCompactSplitSourceTest.java | 57 +
.../procedure/RewriteFileIndexProcedureITCase.java | 9 +
.../procedure/VectorSearchProcedureITCase.java | 205 ++
.../coordinator/TableWriteCoordinatorTest.java | 10 +-
.../apache/paimon/format/avro/AvroBytesArray.java | 380 ++++
.../paimon/format/avro/FieldReaderFactory.java | 17 +-
.../paimon/format/avro/FieldWriterFactory.java | 36 +
.../apache/paimon/format/blob/BlobFileFormat.java | 35 +-
.../paimon/format/blob/BlobFormatReader.java | 160 +-
.../paimon/format/blob/BlobFormatWriter.java | 249 ++-
.../reader/ParquetVectorUpdaterFactory.java | 85 +-
.../paimon/format/avro/AvroBytesArrayTest.java | 335 ++++
.../paimon/format/blob/BlobFileFormatTest.java | 206 ++
.../paimon/format/blob/BlobFormatWriterTest.java | 304 ++-
.../reader/FileTypeNotMatchReadTypeTest.java | 78 +
.../paimon/hive/mapred/PaimonRecordReader.java | 2 +-
paimon-python/dev/run_mixed_tests.sh | 53 +-
.../pypaimon/common/options/core_options.py | 30 +-
paimon-python/pypaimon/daft/daft_blob.py | 42 +
paimon-python/pypaimon/daft/daft_datasource.py | 69 +-
paimon-python/pypaimon/multimodal/blob_store.py | 3 +-
paimon-python/pypaimon/multimodal/query.py | 3 +-
paimon-python/pypaimon/multimodal/table.py | 4 +-
paimon-python/pypaimon/ray/ray_paimon.py | 38 +-
paimon-python/pypaimon/ray/shuffle.py | 20 +-
paimon-python/pypaimon/ray/update_by_row_id.py | 3 +-
.../pypaimon/read/reader/concat_batch_reader.py | 68 +-
.../pypaimon/read/reader/format_blob_reader.py | 249 ++-
.../pypaimon/schema/column_directive_utils.py | 32 +-
paimon-python/pypaimon/schema/data_types.py | 28 +-
paimon-python/pypaimon/schema/schema_manager.py | 117 +-
.../pypaimon/snapshot/snapshot_manager.py | 70 +-
paimon-python/pypaimon/table/row/blob.py | 9 +
.../pypaimon/table/system/partitions_table.py | 42 +
paimon-python/pypaimon/tests/blob_table_test.py | 229 ++-
paimon-python/pypaimon/tests/blob_test.py | 513 ++++-
.../pypaimon/tests/column_directive_utils_test.py | 25 +
.../pypaimon/tests/daft/daft_blob_test.py | 27 +-
.../pypaimon/tests/daft/daft_explain_test.py | 33 +
.../pypaimon/tests/daft/daft_sink_test.py | 54 +
paimon-python/pypaimon/tests/data_types_test.py | 15 +
.../pypaimon/tests/e2e/java_py_read_write_test.py | 74 +
.../pypaimon/tests/filesystem_catalog_test.py | 132 ++
.../pypaimon/tests/ray_read_by_row_id_test.py | 20 +-
.../pypaimon/tests/ray_repartition_test.py | 101 +
.../rest/rest_catalog_commit_snapshot_test.py | 29 +
paimon-python/pypaimon/tests/rest/rest_server.py | 18 +-
.../pypaimon/tests/schema_manager_test.py | 60 +
.../pypaimon/tests/snapshot_manager_test.py | 63 +-
.../pypaimon/tests/system/partitions_table_test.py | 67 +-
paimon-python/pypaimon/write/blob_format_writer.py | 186 +-
paimon-python/pypaimon/write/file_store_write.py | 8 +-
paimon-python/pypaimon/write/ray_datasink.py | 120 ++
paimon-python/pypaimon/write/row_utils.py | 23 +-
.../pypaimon/write/table_update_by_row_id.py | 27 +-
paimon-python/pypaimon/write/table_write.py | 17 +-
.../pypaimon/write/writer/blob_file_writer.py | 42 +-
paimon-python/pypaimon/write/writer/blob_writer.py | 2 +-
.../write/writer/dedicated_format_writer.py | 65 +-
.../scala/org/apache/paimon/spark/SparkTable.scala | 2 +
.../scala/org/apache/paimon/spark/SparkTable.scala | 2 +
.../scala/org/apache/paimon/spark/SparkTable.scala | 2 +
.../MergeIntoPaimonDataEvolutionTable.scala | 48 +-
.../spark/write/PaimonDeltaBatchWrite.scala} | 31 +-
.../apache/spark/sql/paimon/shims/Spark4Shim.scala | 10 +-
.../paimon/spark/AbstractSparkInternalRow.java | 14 +-
.../org/apache/paimon/spark/DataConverter.java | 19 +-
.../java/org/apache/paimon/spark/SparkCatalog.java | 32 +-
.../paimon/spark/SparkInternalRowWrapper.java | 60 +-
.../java/org/apache/paimon/spark/SparkRow.java | 26 +-
.../spark/procedure/RewriteFileIndexProcedure.java | 3 +
...Impl.java => SparkDataEvolutionVectorRead.java} | 10 +-
.../spark/read/SparkPrimaryKeyVectorRead.java | 133 ++
.../spark/read/SparkVectorSearchBuilderImpl.java | 9 +-
.../org/apache/paimon/spark/PaimonBaseScan.scala | 2 +-
.../org/apache/paimon/spark/PaimonMetrics.scala | 10 +
.../paimon/spark/PaimonRecordReaderIterator.scala | 5 +-
.../apache/paimon/spark/PaimonScanBuilder.scala | 5 +
.../apache/paimon/spark/PaimonSparkTableBase.scala | 19 +-
.../scala/org/apache/paimon/spark/SparkTable.scala | 49 +-
.../paimon/spark/SparkV2FilterConverter.scala | 13 +-
.../spark/catalyst/analysis/PaimonAnalysis.scala | 4 +
.../spark/commands/DataEvolutionPaimonWriter.scala | 5 +-
.../MergeIntoPaimonDataEvolutionTable.scala | 48 +-
.../apache/paimon/spark/data/SparkArrayData.scala | 63 +-
.../paimon/spark/data/SparkInternalRow.scala | 4 +-
.../procedure/SparkPostponeCompactProcedure.scala | 231 ++-
.../spark/rowops/PaimonSparkDeltaOperation.scala | 84 +
.../paimon/spark/schema/PaimonMetadataColumn.scala | 9 +-
.../spark/write/DataEvolutionTableDataWrite.scala | 18 +-
.../paimon/spark/write/PaimonBatchWriteBase.scala | 26 +-
.../paimon/spark/write/PaimonDeltaWrite.scala | 77 +
.../paimon/spark/write/PaimonDeltaWriteBase.scala | 225 +++
.../PaimonDeltaWriteBuilder.scala} | 40 +-
.../paimon/spark/write/PaimonDeltaWriter.scala | 150 ++
.../apache/paimon/spark/write/WriteHelper.scala | 21 +
.../apache/spark/sql/paimon/shims/SparkShim.scala | 13 +
....java => SparkDataEvolutionVectorReadTest.java} | 10 +-
.../procedure/RewriteFileIndexProcedureTest.scala | 7 +
.../read/BatchReadTagConcurrentExpireTest.scala | 4 +-
.../org/apache/paimon/spark/sql/BlobTestBase.scala | 211 +-
.../paimon/spark/sql/BlobUpdateTestBase.scala | 21 +
.../org/apache/paimon/spark/sql/DDLTestBase.scala | 15 +
.../paimon/spark/sql/DeletionVectorTest.scala | 76 +-
.../paimon/spark/sql/DeltaRowLevelOpsTest.scala | 302 +++
.../apache/paimon/spark/sql/PaimonMetricTest.scala | 31 +
.../spark/sql/PrimaryKeyVectorSearchTest.scala | 380 ++++
.../spark/sql/SparkV2FilterConverterTestBase.scala | 32 +
.../paimon/spark/sql/VectorSearchOptionsTest.scala | 9 +
.../spark/data/Spark3InternalRowWithBlob.scala | 2 +
.../PaimonDeltaBatchWrite.scala} | 28 +-
.../apache/spark/sql/paimon/shims/Spark3Shim.scala | 10 +-
.../spark/data/Spark4InternalRowWithBlob.scala | 2 +
.../spark/write/PaimonDeltaBatchWrite.scala} | 32 +-
.../catalyst/analysis/PureAppendOnlyScope.scala | 22 +-
.../analysis/Spark41DeleteMetadataRestore.scala | 34 +-
.../analysis/Spark41MergeIntoRewrite.scala | 191 +-
.../analysis/Spark41UpdateTableRewrite.scala | 77 +-
.../apache/spark/sql/paimon/shims/Spark4Shim.scala | 10 +-
pom.xml | 15 +
341 files changed, 32205 insertions(+), 1957 deletions(-)
create mode 100644 docs/docs/multimodal-table/variant.mdx
create mode 100644 docs/docs/primary-key-table/vector-index.md
create mode 100644
paimon-benchmark/paimon-micro-benchmarks/src/test/java/org/apache/paimon/benchmark/compact/ArrayCompactBenchmark.java
copy paimon-core/src/main/java/org/apache/paimon/utils/PartialRow.java =>
paimon-common/src/main/java/org/apache/paimon/data/BlobArrayPlaceholder.java
(53%)
copy
paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/ReaderConsumeProgressEvent.java
=>
paimon-common/src/main/java/org/apache/paimon/data/BlobFetchMetricReporter.java
(52%)
create mode 100644
paimon-common/src/main/java/org/apache/paimon/reader/LimitRecordReader.java
copy
paimon-common/src/main/java/org/apache/paimon/reader/{FileRecordReader.java =>
ScoreRecordReader.java} (81%)
create mode 100644
paimon-common/src/test/java/org/apache/paimon/globalindex/testvector/TestMultiFieldVectorGlobalIndexer.java
create mode 100644
paimon-core/src/main/java/org/apache/paimon/globalindex/sorted/SortedSingleColumnIndexWriter.java
create mode 100644
paimon-core/src/main/java/org/apache/paimon/index/pk/BucketedPrimaryKeyIndexMaintainer.java
copy
paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalogLockContext.java
=>
paimon-core/src/main/java/org/apache/paimon/index/pk/PrimaryKeyIndexDefinition.java
(53%)
create mode 100644
paimon-core/src/main/java/org/apache/paimon/index/pk/PrimaryKeyIndexDefinitions.java
rename
paimon-core/src/main/java/org/apache/paimon/index/{pkvector/PkVectorSourceFile.java
=> pk/PrimaryKeyIndexSourceFile.java} (84%)
rename
paimon-core/src/main/java/org/apache/paimon/index/{pkvector/PkVectorSourceMeta.java
=> pk/PrimaryKeyIndexSourceMeta.java} (52%)
rename
paimon-core/src/main/java/org/apache/paimon/index/{pkvector/PkVectorSourcePolicy.java
=> pk/PrimaryKeyIndexSourcePolicy.java} (85%)
create mode 100644
paimon-core/src/main/java/org/apache/paimon/index/pksorted/BucketedSortedIndexMaintainer.java
create mode 100644
paimon-core/src/main/java/org/apache/paimon/index/pksorted/PkSortedBucketIndexState.java
copy
paimon-core/src/main/java/org/apache/paimon/index/{pkvector/PkVectorDataFileReader.java
=> pksorted/PkSortedDataFileReader.java} (59%)
create mode 100644
paimon-core/src/main/java/org/apache/paimon/index/pksorted/PkSortedIndexBuilder.java
create mode 100644
paimon-core/src/main/java/org/apache/paimon/index/pksorted/PkSortedIndexFile.java
create mode 100644
paimon-core/src/main/java/org/apache/paimon/index/pksorted/PkSortedIndexGroup.java
create mode 100644
paimon-core/src/main/java/org/apache/paimon/index/pkvector/PkVectorAnnLevels.java
create mode 100644
paimon-core/src/main/java/org/apache/paimon/operation/PrimaryKeyIndexedSplitRead.java
create mode 100644
paimon-core/src/main/java/org/apache/paimon/operation/metrics/BlobFetchMetrics.java
rename
paimon-core/src/main/java/org/apache/paimon/table/source/{DataTableBatchScan.java
=> AbstractBatchTableScan.java} (85%)
copy
paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/CdcRecordChannelComputer.java
=>
paimon-core/src/main/java/org/apache/paimon/table/source/AppendBatchTableScan.java
(59%)
create mode 100644
paimon-core/src/main/java/org/apache/paimon/table/source/BucketVectorSearchSplit.java
rename
paimon-core/src/main/java/org/apache/paimon/table/source/{VectorReadImpl.java
=> DataEvolutionVectorRead.java} (96%)
rename
paimon-core/src/main/java/org/apache/paimon/table/source/{VectorScanImpl.java
=> DataEvolutionVectorScan.java} (89%)
copy
paimon-core/src/main/java/org/apache/paimon/table/source/{FullTextRead.java =>
GlobalIndexSplitResult.java} (79%)
create mode 100644
paimon-core/src/main/java/org/apache/paimon/table/source/PrimaryKeyBatchScan.java
create mode 100644
paimon-core/src/main/java/org/apache/paimon/table/source/PrimaryKeySortedIndexResult.java
create mode 100644
paimon-core/src/main/java/org/apache/paimon/table/source/PrimaryKeySortedIndexScan.java
create mode 100644
paimon-core/src/main/java/org/apache/paimon/table/source/PrimaryKeyVectorPositionReader.java
create mode 100644
paimon-core/src/main/java/org/apache/paimon/table/source/PrimaryKeyVectorRead.java
create mode 100644
paimon-core/src/main/java/org/apache/paimon/table/source/PrimaryKeyVectorResult.java
create mode 100644
paimon-core/src/main/java/org/apache/paimon/table/source/PrimaryKeyVectorScan.java
create mode 100644
paimon-core/src/main/java/org/apache/paimon/table/source/VectorSearchRefineOptions.java
copy
paimon-core/src/main/java/org/apache/paimon/table/source/splitread/{RawFileSplitReadProvider.java
=> PrimaryKeyIndexedSplitReadProvider.java} (63%)
create mode 100644
paimon-core/src/test/java/org/apache/paimon/index/pk/BucketedPrimaryKeyIndexMaintainerTest.java
create mode 100644
paimon-core/src/test/java/org/apache/paimon/index/pk/PrimaryKeyIndexDefinitionsTest.java
create mode 100644
paimon-core/src/test/java/org/apache/paimon/index/pk/PrimaryKeyIndexSourceMetaTest.java
create mode 100644
paimon-core/src/test/java/org/apache/paimon/index/pksorted/BucketedSortedIndexMaintainerTest.java
create mode 100644
paimon-core/src/test/java/org/apache/paimon/index/pksorted/PkSortedBucketIndexStateTest.java
copy
paimon-core/src/test/java/org/apache/paimon/index/{pkvector/PkVectorDataFileReaderTest.java
=> pksorted/PkSortedDataFileReaderTest.java} (74%)
create mode 100644
paimon-core/src/test/java/org/apache/paimon/index/pksorted/PkSortedIndexBuilderTest.java
create mode 100644
paimon-core/src/test/java/org/apache/paimon/index/pksorted/PkSortedIndexFileTest.java
create mode 100644
paimon-core/src/test/java/org/apache/paimon/index/pksorted/PrimaryKeySortedIndexMaintenanceTest.java
create mode 100644
paimon-core/src/test/java/org/apache/paimon/index/pksorted/PrimaryKeySortedIndexOptionsTest.java
create mode 100644
paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorAnnLevelsTest.java
delete mode 100644
paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorSourceMetaTest.java
create mode 100644
paimon-core/src/test/java/org/apache/paimon/operation/PrimaryKeyIndexWriteTest.java
create mode 100644
paimon-core/src/test/java/org/apache/paimon/operation/PrimaryKeyIndexedSplitReadTest.java
create mode 100644
paimon-core/src/test/java/org/apache/paimon/operation/metrics/BlobFetchMetricsTest.java
create mode 100644
paimon-core/src/test/java/org/apache/paimon/postpone/BucketFilesTest.java
create mode 100644
paimon-core/src/test/java/org/apache/paimon/schema/PrimaryKeySortedIndexValidationTest.java
create mode 100644
paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeySortedIndexBatchScanTest.java
create mode 100644
paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeySortedIndexReadTest.java
create mode 100644
paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeySortedIndexResultTest.java
create mode 100644
paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeySortedIndexScanTest.java
create mode 100644
paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeyVectorPositionReaderTest.java
create mode 100644
paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeyVectorRawFileSplitReadTest.java
create mode 100644
paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeyVectorReadTest.java
create mode 100644
paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeyVectorResultTest.java
create mode 100644
paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeyVectorScanTest.java
create mode 100644
paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeyVectorSearchTest.java
create mode 100644
paimon-core/src/test/java/org/apache/paimon/table/source/VectorSearchRefineOptionsTest.java
copy
paimon-common/src/test/java/org/apache/paimon/globalindex/sorted/SortedIndexOptionsTest.java
=> paimon-docs/src/test/java/org/apache/paimon/docs/util/UtilsTest.java (55%)
create mode 100644 paimon-eslib/README.md
create mode 100644 paimon-eslib/pom.xml
copy
paimon-format/src/main/java/org/apache/paimon/format/parquet/writer/MetadataParquetBuilder.java
=>
paimon-eslib/src/main/java/org/apache/paimon/eslib/index/ESIndexBuilderFactory.java
(54%)
create mode 100644
paimon-eslib/src/main/java/org/apache/paimon/eslib/index/ESIndexFileMeta.java
create mode 100644
paimon-eslib/src/main/java/org/apache/paimon/eslib/index/ESIndexGlobalIndexReader.java
create mode 100644
paimon-eslib/src/main/java/org/apache/paimon/eslib/index/ESIndexGlobalIndexWriter.java
create mode 100644
paimon-eslib/src/main/java/org/apache/paimon/eslib/index/ESIndexGlobalIndexer.java
copy
paimon-common/src/test/java/org/apache/paimon/globalindex/testfulltext/TestFullTextGlobalIndexerFactory.java
=>
paimon-eslib/src/main/java/org/apache/paimon/eslib/index/ESIndexGlobalIndexerFactory.java
(57%)
create mode 100644
paimon-eslib/src/main/java/org/apache/paimon/eslib/index/ESIndexOptions.java
copy
paimon-common/src/test/resources/META-INF/services/org.apache.paimon.factories.Factory
=>
paimon-eslib/src/main/resources/META-INF/services/org.apache.paimon.globalindex.GlobalIndexerFactory
(93%)
create mode 100644
paimon-eslib/src/test/java/org/apache/paimon/eslib/index/ESIndexFileMetaTest.java
create mode 100644
paimon-eslib/src/test/java/org/apache/paimon/eslib/index/ESIndexFullTextQueryParserTest.java
create mode 100644
paimon-eslib/src/test/java/org/apache/paimon/eslib/index/ESIndexGlobalIndexE2ETest.java
create mode 100644
paimon-eslib/src/test/java/org/apache/paimon/eslib/index/ESIndexGlobalIndexReaderDebugTest.java
create mode 100644
paimon-eslib/src/test/java/org/apache/paimon/eslib/index/ESIndexGlobalIndexerExecutorTest.java
create mode 100644
paimon-eslib/src/test/java/org/apache/paimon/eslib/index/ESIndexOptionsTest.java
create mode 100644
paimon-eslib/src/test/java/org/apache/paimon/eslib/index/ESIndexVectorMetricTest.java
create mode 100644
paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/postpone/PostponeBucketCompactOperator.java
create mode 100644
paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/FlinkRowDataWithBlobTest.java
create mode 100644
paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/KeyValueTableReadLimitITCase.java
create mode 100644
paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/postpone/PostponeBucketCompactSplitSourceTest.java
create mode 100644
paimon-format/src/main/java/org/apache/paimon/format/avro/AvroBytesArray.java
create mode 100644
paimon-format/src/test/java/org/apache/paimon/format/avro/AvroBytesArrayTest.java
copy
paimon-spark/{paimon-spark3-common/src/main/scala/org/apache/paimon/spark/format/FormatTableBatchWrite.scala
=>
paimon-spark-4.0/src/main/scala/org/apache/paimon/spark/write/PaimonDeltaBatchWrite.scala}
(58%)
rename
paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/read/{SparkVectorReadImpl.java
=> SparkDataEvolutionVectorRead.java} (97%)
create mode 100644
paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/read/SparkPrimaryKeyVectorRead.java
create mode 100644
paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/rowops/PaimonSparkDeltaOperation.scala
create mode 100644
paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/write/PaimonDeltaWrite.scala
create mode 100644
paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/write/PaimonDeltaWriteBase.scala
copy
paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/{commands/WithFileStoreTable.scala
=> write/PaimonDeltaWriteBuilder.scala} (55%)
create mode 100644
paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/write/PaimonDeltaWriter.scala
rename
paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/read/{SparkVectorReadImplTest.java
=> SparkDataEvolutionVectorReadTest.java} (97%)
create mode 100644
paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/DeltaRowLevelOpsTest.scala
create mode 100644
paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/PrimaryKeyVectorSearchTest.scala
copy
paimon-spark/paimon-spark3-common/src/main/scala/org/apache/paimon/spark/{format/FormatTableBatchWrite.scala
=> write/PaimonDeltaBatchWrite.scala} (64%)
copy
paimon-spark/{paimon-spark3-common/src/main/scala/org/apache/paimon/spark/format/FormatTableBatchWrite.scala
=>
paimon-spark4-common/src/main/scala/org/apache/paimon/spark/write/PaimonDeltaBatchWrite.scala}
(59%)
