This is an automated email from the ASF dual-hosted git repository.
SteNicholas pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/paimon-cpp.git
The following commit(s) were added to refs/heads/main by this push:
new 9277fdf chore: prepare the repository for v0.3.0 (#162)
9277fdf is described below
commit 9277fdf4cd87a9eb8ab4b9c7a766e2884a57b604
Author: Zhang Jiawei <[email protected]>
AuthorDate: Sat Aug 1 16:00:30 2026 +0800
chore: prepare the repository for v0.3.0 (#162)
---
.asf.yaml | 14 ++--
.github/workflows/build_and_test.yaml | 8 +--
.github/workflows/release_candidate.yaml | 9 +++
CMakeLists.txt | 3 +-
CONTRIBUTING.md | 2 +-
LICENSE | 10 ---
NOTICE | 5 --
README.md | 6 +-
docs/source/build_system.rst | 2 +-
docs/source/building.rst | 31 ++-------
docs/source/conf.py | 8 +--
docs/source/examples/benchmark.rst | 8 ++-
docs/source/index.rst | 4 +-
docs/source/user_guide/global_index.rst | 20 ++++--
docs/source/user_guide/manifest_cache.rst | 13 ++--
include/paimon/utils/special_field_ids.h | 12 ++--
scripts/releasing/README.md | 29 ++++-----
scripts/releasing/create_source_release.sh | 2 +-
scripts/releasing/install_smoke/main.cpp | 10 ++-
scripts/releasing/release_rc.sh | 2 +-
scripts/releasing/tests/test_release_tools.py | 85 +++++++++++++++++++++++--
scripts/releasing/verify_release_candidate.sh | 23 +++++--
src/paimon/CMakeLists.txt | 1 -
src/paimon/core/utils/special_field_ids.cpp | 31 ---------
src/paimon/format/parquet/file_reader_wrapper.h | 3 +-
25 files changed, 191 insertions(+), 150 deletions(-)
diff --git a/.asf.yaml b/.asf.yaml
index 96daab1..0a9c4a5 100644
--- a/.asf.yaml
+++ b/.asf.yaml
@@ -32,7 +32,7 @@ github:
enabled_merge_buttons:
merge: false
squash: true
- rebase: true
+ rebase: false
rulesets:
- name: "Branch Protection"
type: branch
@@ -52,17 +52,19 @@ github:
required_status_checks:
- name: "pre-commit"
app_slug: -1
- - name: "gcc-test"
- app_slug: -1
- name: "rat-license-check"
app_slug: -1
- - name: "test-with-sanitizer"
+ - name: "asan-ubsan"
+ app_slug: -1
+ - name: "tsan"
+ app_slug: -1
+ - name: "clang-debug"
app_slug: -1
- name: "clang-release"
app_slug: -1
- - name: "gcc-release"
+ - name: "gcc-debug"
app_slug: -1
- - name: "clang-test"
+ - name: "gcc-release"
app_slug: -1
- name: "gcc8-test"
app_slug: -1
diff --git a/.github/workflows/build_and_test.yaml
b/.github/workflows/build_and_test.yaml
index a015342..ab3f22b 100644
--- a/.github/workflows/build_and_test.yaml
+++ b/.github/workflows/build_and_test.yaml
@@ -51,10 +51,10 @@ jobs:
cc: gcc-14
cxx: g++-14
- name: clang-debug
- # fetch_depth: '0' # fetch the PR target branch history for
clang-tidy
- # build_args: >-
- # --check_clang_tidy
- # --lint_git_target_commit "origin/${{ github.base_ref ||
github.event.repository.default_branch }}"
+ fetch_depth: '0' # fetch the PR target branch history for
clang-tidy
+ build_args: >-
+ --check_clang_tidy
+ --lint_git_target_commit "origin/${{ github.base_ref ||
github.event.repository.default_branch }}"
- name: asan-ubsan
build_args: --enable_asan --enable_ubsan
- name: tsan
diff --git a/.github/workflows/release_candidate.yaml
b/.github/workflows/release_candidate.yaml
index aaba9ec..28a4b09 100644
--- a/.github/workflows/release_candidate.yaml
+++ b/.github/workflows/release_candidate.yaml
@@ -122,6 +122,15 @@ jobs:
name: source-archive
path: release/ci
+ - name: Install Rust toolchain (tantivy-fts)
+ shell: bash
+ run: ci/scripts/setup_rust.sh
+
+ - name: Install HTTP and TLS development dependencies
+ run: |
+ sudo apt-get update
+ sudo apt-get install -y libcurl4-openssl-dev libssl-dev
+
- name: Verify, build, test, and install
shell: bash
env:
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 52053d4..1122884 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -64,6 +64,7 @@ option(PAIMON_ENABLE_JINDO "Whether to enable jindo file
system" OFF)
option(PAIMON_ENABLE_S3 "Whether to enable S3 file system" OFF)
option(PAIMON_ENABLE_NETWORK_TESTS
"Whether to enable tests that access real remote services over the
network" OFF)
+option(PAIMON_ENABLE_LUMINA "Whether to enable lumina vector index" OFF)
option(PAIMON_ENABLE_LUCENE "Whether to enable lucene index" OFF)
option(PAIMON_ENABLE_TANTIVY
"Whether to enable tantivy-fulltext global index (Rust FFI,
experimental)" OFF)
@@ -485,7 +486,7 @@ include(CMakePackageConfigHelpers)
write_basic_package_version_file(
"${CMAKE_CURRENT_BINARY_DIR}/PaimonConfigVersion.cmake"
VERSION ${PROJECT_VERSION}
- COMPATIBILITY AnyNewerVersion)
+ COMPATIBILITY SameMinorVersion)
configure_package_config_file("${CMAKE_CURRENT_SOURCE_DIR}/PaimonConfig.cmake.in"
"${CMAKE_CURRENT_BINARY_DIR}/PaimonConfig.cmake"
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index cabeca5..3e24673 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -80,7 +80,7 @@ Before submitting, please verify:
### Prerequisites
- **C++17** compatible compiler (GCC recommended)
-- **CMake** ≥ 3.16
+- **CMake** ≥ 3.22
- **Python 3** (for linting scripts and pre-commit)
---
diff --git a/LICENSE b/LICENSE
index 6e39411..bc9931b 100644
--- a/LICENSE
+++ b/LICENSE
@@ -429,16 +429,6 @@ License: https://www.apache.org/licenses/LICENSE-2.0
--------------------------------------------------------------------------------
-This product includes code from LucenePlusPlus.
-
-* LucenePlusPlus utility in src/paimon/global_index/lucene/ directory
-
-Copyright: 2009-2014 Alan Wright.
-Home page: https://github.com/luceneplusplus/LucenePlusPlus
-License: https://www.apache.org/licenses/LICENSE-2.0
-
---------------------------------------------------------------------------------
-
This product includes code derived from PyTorch TH simd.h.
* SIMD detection code in third_party/roaring_bitmap/roaring.cpp
diff --git a/NOTICE b/NOTICE
index a12e432..5510486 100644
--- a/NOTICE
+++ b/NOTICE
@@ -50,10 +50,5 @@ PalDB
--------------------------------------------------------------------------------
-This product includes software from LucenePlusPlus project (Apache 2.0)
-Copyright 2009-2014 Alan Wright.
-
---------------------------------------------------------------------------------
-
JindoSDK NextArch C++
Copyright 2024-present Alibaba Cloud.
diff --git a/README.md b/README.md
index d1140f5..b533ad7 100644
--- a/README.md
+++ b/README.md
@@ -26,10 +26,6 @@ It provides native, high-performance, and extensible access
to the Paimon lake f
Background and documentation are available at
[paimon.apache.org](https://paimon.apache.org).
-## Status
-
-Paimon-cpp is currently undergoing repository migration. The original
repository is hosted at
[github.com/alibaba/paimon-cpp](https://github.com/alibaba/paimon-cpp/), and
the codebase is being migrated incrementally to the Apache Paimon community
repository.
-
## Features
Paimon-cpp currently provides:
@@ -53,7 +49,7 @@ Paimon-cpp currently provides:
> Paimon C++ therefore currently treats the `bitmap` global index type as
> unsupported. The legacy
> implementation remains in the codebase pending migration to the
> Java-compatible format.
-Note: Linux x86_64 and macOS arm64 builds are currently verified.
+Note: Only Linux x86_64 builds are currently supported and verified.
## Building
diff --git a/docs/source/build_system.rst b/docs/source/build_system.rst
index f3b18e5..b2db810 100644
--- a/docs/source/build_system.rst
+++ b/docs/source/build_system.rst
@@ -44,7 +44,7 @@ for data format and file system.
.. code-block:: cmake
- cmake_minimum_required(VERSION 3.16)
+ cmake_minimum_required(VERSION 3.22)
project(MyExample)
diff --git a/docs/source/building.rst b/docs/source/building.rst
index 8608107..1041d57 100644
--- a/docs/source/building.rst
+++ b/docs/source/building.rst
@@ -32,9 +32,8 @@ out-of-source. For example, you could create
``paimon-cpp/build`` and invoke
Building requires:
-* A C++17-enabled compiler. On Linux, gcc 8 and higher should be
- sufficient. On macOS, use AppleClang from Xcode Command Line Tools or
- LLVM clang from Homebrew. Windows is not supported for now.
+* A C++17-enabled compiler. On Linux, gcc 8 and higher should be sufficient.
+ macOS and Windows are not supported for now.
* At least 2GB of RAM for a minimal build, 8GB for a minimal
debug build with tests and 16GB for a full build.
@@ -46,29 +45,6 @@ On Ubuntu/Debian you can install the requirements with:
build-essential \
cmake
-On macOS you can install the requirements with:
-
-.. code-block:: shell
-
- xcode-select --install
- brew install cmake
-
-The same CMake build options apply on Linux and macOS. If you prefer upstream
-LLVM clang instead of AppleClang on macOS, install LLVM and pass the Homebrew
-compiler paths when configuring:
-
-.. code-block:: shell
-
- brew install llvm
- cmake -B build \
- -DCMAKE_C_COMPILER="$(brew --prefix llvm)/bin/clang" \
- -DCMAKE_CXX_COMPILER="$(brew --prefix llvm)/bin/clang++"
-
-When building with upstream Clang on macOS, Paimon uses Apple SDK libc++
-headers by default to avoid incompatibilities in bundled third-party
-dependencies. Pass ``-DPAIMON_USE_APPLE_LIBCXX_WITH_CLANG=OFF`` to disable
-this behavior.
-
We also provide a docker template to help you get started quickly. See in
``.devcontainer`` folder for more details.
@@ -149,6 +125,7 @@ boolean flags to ``cmake``.
* ``-DPAIMON_ENABLE_JINDO=ON``: Support for Alibaba Jindo filesystems
* ``-DPAIMON_ENABLE_LUMINA=ON``: Support for the Lumina vector index.
* ``-DPAIMON_ENABLE_LUCENE=ON``: Support for Lucene full-text search indexes
+* ``-DPAIMON_ENABLE_TANTIVY=ON``: Enable the experimental Tantivy full-text
index Rust FFI.
Third-party dependency source
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -241,7 +218,7 @@ several times with different options if you want to
exercise all of them.
CMake version requirements
~~~~~~~~~~~~~~~~~~~~~~~~~~
-We support CMake 3.16 and higher.
+We support CMake 3.22 and higher.
LLVM and Clang Tools
~~~~~~~~~~~~~~~~~~~~
diff --git a/docs/source/conf.py b/docs/source/conf.py
index 92dfa7c..4b4b778 100644
--- a/docs/source/conf.py
+++ b/docs/source/conf.py
@@ -32,8 +32,8 @@
import os
import datetime
-project = u'C++ Paimon'
-copyright = f'2024-{datetime.datetime.now().year} Apache Software Foundation'
+project = u'Apache Paimon C++'
+copyright = f'2026-{datetime.datetime.now().year} Apache Software Foundation'
author = u'Apache Software Foundation'
# -- General configuration ---------------------------------------------------
@@ -131,7 +131,7 @@ html_theme_options = {
},
],
"logo": {
- "text": "Paimon C++",
+ "text": "Apache Paimon C++",
},
"show_version_warning_banner": True,
}
@@ -143,6 +143,6 @@ html_context = {
"doc_path": "docs/source",
}
-html_title = f"C++ Paimon"
+html_title = "Apache Paimon C++"
html_show_sourcelink = False
diff --git a/docs/source/examples/benchmark.rst
b/docs/source/examples/benchmark.rst
index ee17be7..d2ff1cd 100644
--- a/docs/source/examples/benchmark.rst
+++ b/docs/source/examples/benchmark.rst
@@ -54,9 +54,11 @@ specific options below:
``--paimon_option=<key1>:<value1>;<key2>:<value2>``
Repeatable table options passed through to Paimon. The default table file
- format is ``parquet``; use ``--paimon_option file.format:<format>`` to
- override it. For ``BM_PK_Write`` and ``BM_MOR_Read``, ``bucket`` is forced
to
- ``1``.
+ format is ``parquet``. Benchmark output supports ``parquet`` and, when
+ built with ``PAIMON_ENABLE_ORC=ON``, ``orc``; use
+ ``--paimon_option file.format:parquet`` or
+ ``--paimon_option file.format:orc`` to select one. For ``BM_PK_Write`` and
+ ``BM_MOR_Read``, ``bucket`` is forced to ``1``.
Examples
========
diff --git a/docs/source/index.rst b/docs/source/index.rst
index 5c4d83a..7ad9a1d 100644
--- a/docs/source/index.rst
+++ b/docs/source/index.rst
@@ -17,8 +17,8 @@
.. _implementations:
-C++ Paimon Documentation
-========================
+Apache Paimon C++
+=================
Paimon C++ is a high-performance C++ implementation of Apache Paimon. We aim to
provide a native, high-performance and extensible implementation that allows
diff --git a/docs/source/user_guide/global_index.rst
b/docs/source/user_guide/global_index.rst
index bc12c3a..0cb853a 100644
--- a/docs/source/user_guide/global_index.rst
+++ b/docs/source/user_guide/global_index.rst
@@ -20,12 +20,12 @@ Global Index
Global Index is a powerful indexing mechanism for append-only tables.
It enables efficient row-level lookups and filtering without full-table scans.
-Paimon supports multiple global index types:
+Paimon C++ supports the following global index types:
-- **Bitmap Index**: A bitmap-based index. Each distinct value is mapped to a
compressed bitmap (RoaringBitmap) that records which rows contain that value,
enabling extremely fast set membership tests.
- **BTree Index**: An efficient index based on multi-level SST files for
scalar column lookups.
- **Range Bitmap Index**: A range bitmap index optimized for range predicates
on ordered scalar columns. Extends the bitmap approach by encoding value
ordering, enabling efficient less-than, greater-than, and range conditions.
- **Lucene Index**: A full-text search index powered by Lucene++. Supports
tokenized text search with multiple modes including match-all, match-any,
phrase, prefix, and wildcard queries.
+- **Tantivy Full-Text Index (experimental)**: A Rust FFI-based full-text
search index.
- **Vector Index (Lumina)**: An approximate nearest neighbor (ANN) index
powered by Lumina for vector similarity search with configurable distance
metrics.
Global indexes work on top of Data Evolution tables. To use global indexes,
your table must have:
@@ -34,12 +34,10 @@ Global indexes work on top of Data Evolution tables. To use
global indexes, your
- ``'row-tracking.enabled' = 'true'``
- ``'data-evolution.enabled' = 'true'``
-Bitmap Index
-------------
+Bitmap Index Compatibility
+--------------------------
-A bitmap-based index for Equal and In predicates. Each distinct value in the
indexed column
-is mapped to a compressed bitmap (RoaringBitmap) that records which rows
contain that value.
-This allows extremely fast set membership tests.
+The current Paimon C++ version does not support bitmap global indexes.
BTree Index
-----------
@@ -94,6 +92,14 @@ search modes including match-all, match-any, phrase, prefix,
and wildcard querie
- **Description**: Specifies the directory containing Jieba dictionary files
for Chinese text tokenization. At runtime, the system first checks this
environment variable; if not set, it falls back to the compile-time
``JIEBA_TEST_DICT_DIR`` macro (only available in test builds). If neither is
available, will fail with an error.
+Tantivy Full-Text Index (Experimental)
+---------------------------------------
+
+The Tantivy full-text index is an experimental Rust FFI-based backend. Enable
it at build time
+with ``-DPAIMON_ENABLE_TANTIVY=ON``. Its global index identifier is
``tantivy-fulltext``.
+Because this backend is experimental, validate its behavior and compatibility
for your workload
+before production use.
+
Vector Index (Lumina)
---------------------
diff --git a/docs/source/user_guide/manifest_cache.rst
b/docs/source/user_guide/manifest_cache.rst
index 0e02750..ed01e0f 100644
--- a/docs/source/user_guide/manifest_cache.rst
+++ b/docs/source/user_guide/manifest_cache.rst
@@ -22,10 +22,9 @@ Overview
--------
paimon-cpp caches raw manifest file bytes at the ``ObjectsFile<T>::Read()``
-layer. The cache uses the public ``Cache`` abstraction and is injected through
-``ScanContextBuilder`` or ``ReadContextBuilder``. The cache covers data
-manifests, manifest lists, and index manifests because they all read through
-``ObjectsFile<T>``.
+layer. The cache uses the public ``Cache`` abstraction and is enabled through
+``ScanContextBuilder::WithCache()``. The cache covers data manifests, manifest
+lists, and index manifests because they all read through ``ObjectsFile<T>``.
For repeated ``get``, ``scan``, or batch ``get/scan -f`` requests in the same
process, the same snapshot often reads the same manifest files repeatedly. On a
@@ -84,10 +83,8 @@ Example:
paimon::ScanContextBuilder scan_builder(table_path);
scan_builder.WithCache(cache);
- paimon::ReadContextBuilder read_builder(table_path);
- read_builder.WithCache(cache);
-
-Passing ``nullptr`` or omitting ``WithCache()`` leaves manifest caching
disabled.
+Passing ``nullptr`` or omitting ``ScanContextBuilder::WithCache()`` leaves
+manifest caching disabled.
Future Optimizations
--------------------
diff --git a/include/paimon/utils/special_field_ids.h
b/include/paimon/utils/special_field_ids.h
index accf07f..c4e03f0 100644
--- a/include/paimon/utils/special_field_ids.h
+++ b/include/paimon/utils/special_field_ids.h
@@ -28,20 +28,20 @@ namespace paimon {
class SpecialFieldIds {
protected:
/// System defined constant for field id boundary. Value: INT32_MAX - 10000
- static const int32_t CPP_FIELD_ID_END =
std::numeric_limits<int32_t>::max() - 10000;
+ inline static constexpr int32_t CPP_FIELD_ID_END =
std::numeric_limits<int32_t>::max() - 10000;
public:
/// Special field ID reserved for sequence number. Value: INT32_MAX - 1
- static const int32_t SEQUENCE_NUMBER = std::numeric_limits<int32_t>::max()
- 1;
+ inline static constexpr int32_t SEQUENCE_NUMBER =
std::numeric_limits<int32_t>::max() - 1;
/// Special field ID reserved for value kind. Value: INT32_MAX - 2
- static const int32_t VALUE_KIND = std::numeric_limits<int32_t>::max() - 2;
+ inline static constexpr int32_t VALUE_KIND =
std::numeric_limits<int32_t>::max() - 2;
/// Special field ID reserved for row kind. Value: INT32_MAX - 3
- static const int32_t ROW_KIND = std::numeric_limits<int32_t>::max() - 3;
+ inline static constexpr int32_t ROW_KIND =
std::numeric_limits<int32_t>::max() - 3;
/// Special field ID reserved for row ID. Value: INT32_MAX - 5
- static const int32_t ROW_ID = std::numeric_limits<int32_t>::max() - 5;
+ inline static constexpr int32_t ROW_ID =
std::numeric_limits<int32_t>::max() - 5;
/// Special field ID reserved for index score. Value: CPP_FIELD_ID_END - 1
- static const int32_t INDEX_SCORE = CPP_FIELD_ID_END - 1;
+ inline static constexpr int32_t INDEX_SCORE = CPP_FIELD_ID_END - 1;
};
} // namespace paimon
diff --git a/scripts/releasing/README.md b/scripts/releasing/README.md
index d7757a9..ee558c1 100644
--- a/scripts/releasing/README.md
+++ b/scripts/releasing/README.md
@@ -47,16 +47,15 @@ Before starting a release:
For example, update all version locations and review the diff:
```bash
-scripts/releasing/bump_version.py 0.2.2 0.2.3
-scripts/releasing/bump_version.py --check 0.2.3
+scripts/releasing/bump_version.py 0.2.0 0.3.0
+scripts/releasing/bump_version.py --check 0.3.0
```
### Signing key setup and security
Complete signing-key setup well before creating the first release candidate:
-- new signing keys must use RSA with at least 2048 bits; ASF recommends 4096
- bits for new keys;
+- new signing keys must use RSA with at least 4096 bits;
- publish the public key to the global public keyserver network;
- append the public key to `dist/release/paimon/KEYS`. Never remove historical
keys because they are required to verify archived releases;
@@ -74,8 +73,8 @@ See the ASF
policies for the complete requirements.
The release scripts use `vVERSION-rcRC` for release-candidate tags and
-`vVERSION` for the final release tag. For example, the first 0.2.3 candidate is
-`v0.2.3-rc1`.
+`vVERSION` for the final release tag. For example, the first 0.3.0 candidate is
+`v0.3.0-rc1`.
## Create a release candidate
@@ -88,7 +87,7 @@ workflow to succeed, and imports the artifacts into ASF
`dist/dev`:
```bash
scripts/releasing/release_rc.sh \
- --version 0.2.3 \
+ --version 0.3.0 \
--rc 1 \
--signing-key ASF_GPG_KEY_ID \
--remote upstream
@@ -107,9 +106,9 @@ email and must not be used to start a vote.
The candidate directory contains:
```text
-apache-paimon-cpp-0.2.3-src.tgz
-apache-paimon-cpp-0.2.3-src.tgz.asc
-apache-paimon-cpp-0.2.3-src.tgz.sha512
+apache-paimon-cpp-0.3.0-src.tgz
+apache-paimon-cpp-0.3.0-src.tgz.asc
+apache-paimon-cpp-0.3.0-src.tgz.sha512
```
The wrapper prints a vote-email template. Send it to `[email protected]`.
@@ -140,7 +139,7 @@ After its vote is closed, a failed or superseded candidate
may be removed from
Voters can download and verify an ASF-staged candidate in one command:
```bash
-scripts/releasing/verify_release_candidate.sh --version 0.2.3 --rc 1
+scripts/releasing/verify_release_candidate.sh --version 0.3.0 --rc 1
```
To verify files that were downloaded separately, use an explicitly downloaded
@@ -149,14 +148,14 @@ KEYS file so signature verification runs in an isolated
GPG home:
```bash
scripts/releasing/verify_release_candidate.sh \
--keys-file /path/to/paimon-KEYS \
- apache-paimon-cpp-0.2.3-src.tgz
+ apache-paimon-cpp-0.3.0-src.tgz
```
The verifier checks:
- the SHA-512 checksum and detached OpenPGP signature;
- archive path safety, portable filename collisions, file types, permissions,
- and the single `paimon-cpp-0.2.3/` root directory;
+ and the single `paimon-cpp-0.3.0/` root directory;
- required `LICENSE`, `NOTICE`, build, and documentation files;
- the CMake and documentation versions;
- absence of compiled artifacts by filename and file magic;
@@ -164,7 +163,7 @@ The verifier checks:
- a release build and the test suite from the extracted source archive; and
- installation plus compilation and execution of an external CMake consumer.
-Pass `--git-ref v0.2.3-rc1` when the Git repository is available to regenerate
+Pass `--git-ref v0.3.0-rc1` when the Git repository is available to regenerate
the archive from the signed tag and compare it byte-for-byte.
`--allow-unsigned`, `--skip-rat`, `--skip-build`, and `--skip-install` exist
for
@@ -180,7 +179,7 @@ approved candidate:
```bash
scripts/releasing/publish_release.sh \
- --version 0.2.3 \
+ --version 0.3.0 \
--rc 1 \
--signing-key ASF_GPG_KEY_ID \
--remote upstream \
diff --git a/scripts/releasing/create_source_release.sh
b/scripts/releasing/create_source_release.sh
index f65ae2b..125ab57 100755
--- a/scripts/releasing/create_source_release.sh
+++ b/scripts/releasing/create_source_release.sh
@@ -35,7 +35,7 @@ Usage:
create_source_release.sh --version VERSION --git-ref REF [options]
Required:
- --version VERSION Release version, for example 0.2.3
+ --version VERSION Release version, for example 0.3.0
--git-ref REF Commit or signed RC tag to archive
Options:
diff --git a/scripts/releasing/install_smoke/main.cpp
b/scripts/releasing/install_smoke/main.cpp
index 914adfc..34db4f7 100644
--- a/scripts/releasing/install_smoke/main.cpp
+++ b/scripts/releasing/install_smoke/main.cpp
@@ -16,12 +16,18 @@
* limitations under the License.
*/
+#include <cstdint>
+#include <limits>
#include <string>
#include "paimon/status.h"
+#include "paimon/utils/special_field_ids.h"
int main() {
const paimon::Status status = paimon::Status::Invalid("install smoke
test");
- return !status.ok() && status.ToString().find("install smoke test") !=
std::string::npos ? 0
-
: 1;
+ const int32_t* row_id = &paimon::SpecialFieldIds::ROW_ID;
+ const bool valid_status =
+ !status.ok() && status.ToString().find("install smoke test") !=
std::string::npos;
+ const bool valid_row_id = *row_id == std::numeric_limits<int32_t>::max() -
5;
+ return valid_status && valid_row_id ? 0 : 1;
}
diff --git a/scripts/releasing/release_rc.sh b/scripts/releasing/release_rc.sh
index 6b5f405..f943491 100755
--- a/scripts/releasing/release_rc.sh
+++ b/scripts/releasing/release_rc.sh
@@ -40,7 +40,7 @@ Usage:
release_rc.sh --version VERSION --rc RC --signing-key KEY_ID [options]
Required:
- --version VERSION Release version, for example 0.2.3
+ --version VERSION Release version, for example 0.3.0
--rc RC Release candidate number, for example 1
--signing-key KEY_ID OpenPGP key used for the tag and source artifact
diff --git a/scripts/releasing/tests/test_release_tools.py
b/scripts/releasing/tests/test_release_tools.py
index e3dbdac..d1b4eea 100644
--- a/scripts/releasing/tests/test_release_tools.py
+++ b/scripts/releasing/tests/test_release_tools.py
@@ -19,13 +19,14 @@
import hashlib
import io
import json
+import os
import subprocess
import sys
import tarfile
import tempfile
import unittest
from pathlib import Path
-from typing import List, Optional, Tuple
+from typing import Dict, List, Optional, Tuple
RELEASING_DIR = Path(__file__).resolve().parents[1]
@@ -90,6 +91,17 @@ class ReleaseToolTest(unittest.TestCase):
),
".github/.rat-excludes": b"",
"scripts/releasing/create_source_release.sh": b"#!/usr/bin/env
bash\n",
+ "ci/scripts/build_paimon.sh": b"""#!/usr/bin/env bash
+set -euo pipefail
+source_root=$(cd "$(dirname "$0")/../.." && pwd)
+[[ $# == 5 ]]
+[[ $1 == --source_dir ]]
+[[ $2 == "${source_root}" ]]
+[[ $3 == --build_type ]]
+[[ $4 == Release ]]
+[[ $5 == --install_smoke ]]
+[[ ${PAIMON_BUILD_JOBS} == 7 ]]
+""",
}
with tarfile.open(artifact, mode="w:gz") as archive:
root = tarfile.TarInfo("paimon-cpp-1.2.3/")
@@ -109,20 +121,27 @@ class ReleaseToolTest(unittest.TestCase):
return artifact
def run_verifier(
- self, artifact: Path, *, expected_returncode: int = 0
+ self,
+ artifact: Path,
+ *,
+ options: Optional[List[str]] = None,
+ env: Optional[Dict[str, str]] = None,
+ expected_returncode: int = 0,
) -> subprocess.CompletedProcess:
+ if options is None:
+ options = ["--skip-rat", "--skip-build"]
result = subprocess.run(
[
"bash",
str(RELEASE_VERIFIER),
"--allow-unsigned",
- "--skip-rat",
- "--skip-build",
+ *options,
str(artifact),
],
universal_newlines=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
+ env=env,
check=False,
)
self.assertEqual(
@@ -132,6 +151,24 @@ class ReleaseToolTest(unittest.TestCase):
)
return result
+ def create_fake_rat(self, directory: Path) -> Tuple[Path, Dict[str, str]]:
+ rat_jar = directory / "apache-rat.jar"
+ rat_jar.touch()
+ bin_dir = directory / "bin"
+ bin_dir.mkdir()
+ java = bin_dir / "java"
+ java.write_text(
+ "#!/usr/bin/env bash\n"
+ "set -euo pipefail\n"
+ "printf '%s Unknown Licenses\\n' \"${FAKE_RAT_UNKNOWN_COUNT:?}\"\n"
+ "printf '%s\\n' 'Files with unapproved licenses:'\n",
+ encoding="utf-8",
+ )
+ java.chmod(0o755)
+ env = os.environ.copy()
+ env["PATH"] = f"{bin_dir}{os.pathsep}{env['PATH']}"
+ return rat_jar, env
+
def test_archive_validator_accepts_regular_archive(self) -> None:
with tempfile.TemporaryDirectory() as temp:
artifact = Path(temp) / "valid.tgz"
@@ -237,6 +274,46 @@ class ReleaseToolTest(unittest.TestCase):
result.stderr,
)
+ def test_verifier_invokes_build_script_with_named_arguments(self) -> None:
+ with tempfile.TemporaryDirectory() as temp:
+ artifact = self.create_verifier_archive(Path(temp))
+ result = self.run_verifier(
+ artifact,
+ options=["--skip-rat", "--jobs", "7"],
+ )
+ self.assertIn("Release build and tests: valid", result.stdout)
+ self.assertIn("Install and consumer smoke test: valid",
result.stdout)
+
+ def test_verifier_accepts_zero_unknown_licenses(self) -> None:
+ with tempfile.TemporaryDirectory() as temp:
+ directory = Path(temp)
+ artifact = self.create_verifier_archive(directory)
+ rat_jar, env = self.create_fake_rat(directory)
+ env["FAKE_RAT_UNKNOWN_COUNT"] = "0"
+ result = self.run_verifier(
+ artifact,
+ options=["--rat-jar", str(rat_jar), "--skip-build"],
+ env=env,
+ )
+ self.assertIn("Apache RAT: valid", result.stdout)
+
+ def test_verifier_rejects_unknown_licenses(self) -> None:
+ with tempfile.TemporaryDirectory() as temp:
+ directory = Path(temp)
+ artifact = self.create_verifier_archive(directory)
+ rat_jar, env = self.create_fake_rat(directory)
+ env["FAKE_RAT_UNKNOWN_COUNT"] = "1"
+ result = self.run_verifier(
+ artifact,
+ options=["--rat-jar", str(rat_jar), "--skip-build"],
+ env=env,
+ expected_returncode=1,
+ )
+ self.assertIn(
+ "Apache RAT found 1 files with unknown licenses",
+ result.stderr,
+ )
+
def create_version_tree(self, root: Path) -> None:
(root / "docs/source/_static").mkdir(parents=True)
(root / "CMakeLists.txt").write_text(
diff --git a/scripts/releasing/verify_release_candidate.sh
b/scripts/releasing/verify_release_candidate.sh
index dee7d81..99e7df0 100755
--- a/scripts/releasing/verify_release_candidate.sh
+++ b/scripts/releasing/verify_release_candidate.sh
@@ -339,9 +339,18 @@ if [[ "${SKIP_RAT}" == false ]]; then
-E "${SOURCE_DIR}/.github/.rat-excludes" \
-d "${SOURCE_DIR}" >"${RAT_REPORT}"
- if grep -Eq 'Files with unapproved licenses:[[:space:]]*[1-9]'
"${RAT_REPORT}"; then
+ UNKNOWN_LICENSE_COUNT=$(
+ awk '/^[[:space:]]*[0-9]+[[:space:]]+Unknown Licenses[[:space:]]*$/ {
+ print $1
+ }' "${RAT_REPORT}"
+ )
+ if [[ ! "${UNKNOWN_LICENSE_COUNT}" =~ ^[0-9]+$ ]]; then
cat "${RAT_REPORT}"
- fail "Apache RAT found files with unapproved licenses"
+ fail "could not determine the Apache RAT unknown license count"
+ fi
+ if [[ "${UNKNOWN_LICENSE_COUNT}" != "0" ]]; then
+ cat "${RAT_REPORT}"
+ fail "Apache RAT found ${UNKNOWN_LICENSE_COUNT} files with unknown
licenses"
fi
echo "Apache RAT: valid"
else
@@ -353,9 +362,15 @@ if [[ "${SKIP_BUILD}" == false ]]; then
if [[ "${SKIP_INSTALL}" == true ]]; then
INSTALL_SMOKE=false
fi
+ BUILD_ARGS=(
+ --source_dir "${SOURCE_DIR}"
+ --build_type Release
+ )
+ if [[ "${INSTALL_SMOKE}" == true ]]; then
+ BUILD_ARGS+=(--install_smoke)
+ fi
PAIMON_BUILD_JOBS="${JOBS}" \
- "${SOURCE_DIR}/ci/scripts/build_paimon.sh" \
- "${SOURCE_DIR}" false false Release "${INSTALL_SMOKE}"
+ "${SOURCE_DIR}/ci/scripts/build_paimon.sh" "${BUILD_ARGS[@]}"
echo "Release build and tests: valid"
if [[ "${INSTALL_SMOKE}" == true ]]; then
echo "Install and consumer smoke test: valid"
diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt
index f80c3ee..ce2f5dd 100644
--- a/src/paimon/CMakeLists.txt
+++ b/src/paimon/CMakeLists.txt
@@ -403,7 +403,6 @@ set(PAIMON_CORE_SRCS
core/utils/partition_path_utils.cpp
core/utils/primary_key_table_utils.cpp
core/utils/snapshot_manager.cpp
- core/utils/special_field_ids.cpp
core/utils/tag_manager.cpp)
add_paimon_lib(paimon
diff --git a/src/paimon/core/utils/special_field_ids.cpp
b/src/paimon/core/utils/special_field_ids.cpp
deleted file mode 100644
index 7b07837..0000000
--- a/src/paimon/core/utils/special_field_ids.cpp
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#include "paimon/utils/special_field_ids.h"
-
-namespace paimon {
-
-const int32_t SpecialFieldIds::CPP_FIELD_ID_END;
-const int32_t SpecialFieldIds::SEQUENCE_NUMBER;
-const int32_t SpecialFieldIds::VALUE_KIND;
-const int32_t SpecialFieldIds::ROW_KIND;
-const int32_t SpecialFieldIds::ROW_ID;
-const int32_t SpecialFieldIds::INDEX_SCORE;
-
-} // namespace paimon
diff --git a/src/paimon/format/parquet/file_reader_wrapper.h
b/src/paimon/format/parquet/file_reader_wrapper.h
index c62c1b9..5dbd381 100644
--- a/src/paimon/format/parquet/file_reader_wrapper.h
+++ b/src/paimon/format/parquet/file_reader_wrapper.h
@@ -69,7 +69,8 @@ class FileReaderWrapper {
/// @return The next RecordBatch, or nullptr if end of data.
Result<std::shared_ptr<arrow::RecordBatch>> Next();
- /// Get the first row number of the previously returned batch.
+ /// Get the first row number of the previously returned batch. After
Next() reaches EOF,
+ /// returns the next unread row number (the end of the readable range).
Result<uint64_t> GetPreviousBatchFirstRowNumber() const {
return previous_first_row_;
}
