This is an automated email from the ASF dual-hosted git repository.
lxy-9602 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/paimon-cpp.git
The following commit(s) were added to refs/heads/main by this push:
new d89d8495 feat(file-index): support additional types and validate
decimal scale (#271)
d89d8495 is described below
commit d89d8495883e80a44e0e7b200d7bb6e6fba3a153
Author: Zhang Jiawei <[email protected]>
AuthorDate: Wed Sep 9 19:01:18 2026 +0800
feat(file-index): support additional types and validate decimal scale (#271)
---
include/paimon/file_index/file_index_reader.h | 2 +
include/paimon/global_index/global_index_reader.h | 2 +
src/paimon/CMakeLists.txt | 1 +
src/paimon/common/file_index/CMakeLists.txt | 4 +-
.../file_index/bitmap/bitmap_file_index_meta.cpp | 32 +++
.../file_index/bitmap/bitmap_file_index_test.cpp | 245 +++++++++++++++++++++
.../bsi/bit_slice_index_bitmap_file_index.cpp | 21 +-
.../bsi/bit_slice_index_bitmap_file_index_test.cpp | 26 +++
.../rangebitmap/dictionary/chunked_dictionary.cpp | 6 +-
.../rangebitmap/dictionary/chunked_dictionary.h | 1 -
.../dictionary/chunked_dictionary_test.cpp | 22 +-
.../rangebitmap/dictionary/key_factory.cpp | 26 ++-
.../dictionary/variable_length_chunk.cpp | 171 ++++++++++++++
.../rangebitmap/dictionary/variable_length_chunk.h | 103 +++++++++
.../rangebitmap/range_bitmap_file_index.cpp | 89 +++++---
.../rangebitmap/range_bitmap_file_index.h | 20 +-
.../rangebitmap/range_bitmap_file_index_test.cpp | 187 +++++++++++++++-
.../rangebitmap/range_bitmap_type_adapter.cpp | 113 ++++++++++
.../rangebitmap/range_bitmap_type_adapter.h | 56 +++++
.../rangebitmap/range_bitmap_type_adapter_test.cpp | 109 +++++++++
.../btree/btree_global_index_integration_test.cpp | 8 +
src/paimon/common/predicate/predicate_validator.h | 50 ++++-
.../common/predicate/predicate_validator_test.cpp | 104 +++++++--
.../core/operation/internal_read_context.cpp | 2 -
src/paimon/core/table/format/format_table_read.cpp | 1 -
src/paimon/core/table/source/table_scan.cpp | 2 -
test/inte/blob_table_inte_test.cpp | 5 +-
test/inte/global_index_test.cpp | 8 +
test/inte/read_inte_test.cpp | 2 +-
test/inte/scan_inte_test.cpp | 2 +-
30 files changed, 1332 insertions(+), 88 deletions(-)
diff --git a/include/paimon/file_index/file_index_reader.h
b/include/paimon/file_index/file_index_reader.h
index ff1d31d1..2da36fd2 100644
--- a/include/paimon/file_index/file_index_reader.h
+++ b/include/paimon/file_index/file_index_reader.h
@@ -34,6 +34,8 @@ namespace paimon {
/// `std::shared_ptr<FileIndexResult>` objects. It reads pre-built file-level
index data
/// (e.g., bitmap, bsi or bloom filters) from index file and evaluates
/// whether a given data file may contain rows matching a specific predicate.
+/// @note Callers of `Visit*` for DECIMAL fields must ensure each literal's
scale matches the scale
+/// of the indexed data; otherwise, index filtering results may be incorrect.
class PAIMON_EXPORT FileIndexReader : public
FunctionVisitor<std::shared_ptr<FileIndexResult>> {
public:
Result<std::shared_ptr<FileIndexResult>> VisitIsNotNull() override;
diff --git a/include/paimon/global_index/global_index_reader.h
b/include/paimon/global_index/global_index_reader.h
index ca3f4c9b..25792465 100644
--- a/include/paimon/global_index/global_index_reader.h
+++ b/include/paimon/global_index/global_index_reader.h
@@ -35,6 +35,8 @@ namespace paimon {
/// Derived classes are expected to implement the visitor methods (e.g.,
`VisitEqual`,
/// `VisitIsNull`, etc.) to return index-based results that indicate which
/// rows satisfy the given predicate.
+/// @note Callers of `Visit*` for DECIMAL fields must ensure each literal's
scale matches the scale
+/// of the indexed data; otherwise, index filtering results may be incorrect.
class PAIMON_EXPORT GlobalIndexReader : public
FunctionVisitor<std::shared_ptr<GlobalIndexResult>> {
public:
/// VisitVectorSearch performs approximate vector similarity search.
diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt
index 0e7433a0..6fc1deaa 100644
--- a/src/paimon/CMakeLists.txt
+++ b/src/paimon/CMakeLists.txt
@@ -591,6 +591,7 @@ if(PAIMON_BUILD_TESTS)
common/file_index/rangebitmap/dictionary/chunked_dictionary_test.cpp
common/file_index/rangebitmap/range_bitmap_file_index_test.cpp
common/file_index/rangebitmap/range_bitmap_io_test.cpp
+
common/file_index/rangebitmap/range_bitmap_type_adapter_test.cpp
common/file_index/bloomfilter/bloom_filter_file_index_test.cpp
common/file_index/bloomfilter/fast_hash_test.cpp
common/global_index/complete_index_score_batch_reader_test.cpp
diff --git a/src/paimon/common/file_index/CMakeLists.txt
b/src/paimon/common/file_index/CMakeLists.txt
index 7ab5a069..1e083c51 100644
--- a/src/paimon/common/file_index/CMakeLists.txt
+++ b/src/paimon/common/file_index/CMakeLists.txt
@@ -29,11 +29,13 @@ set(PAIMON_FILE_INDEX_SRC
rangebitmap/dictionary/chunked_dictionary.cpp
rangebitmap/dictionary/fixed_length_chunk.cpp
rangebitmap/dictionary/key_factory.cpp
+ rangebitmap/dictionary/variable_length_chunk.cpp
rangebitmap/utils/literal_serialization_utils.cpp
rangebitmap/bit_slice_index_bitmap.cpp
rangebitmap/range_bitmap.cpp
rangebitmap/range_bitmap_file_index.cpp
- rangebitmap/range_bitmap_file_index_factory.cpp)
+ rangebitmap/range_bitmap_file_index_factory.cpp
+ rangebitmap/range_bitmap_type_adapter.cpp)
add_paimon_lib(paimon_file_index
SOURCES
diff --git a/src/paimon/common/file_index/bitmap/bitmap_file_index_meta.cpp
b/src/paimon/common/file_index/bitmap/bitmap_file_index_meta.cpp
index 64e05c65..50b88af3 100644
--- a/src/paimon/common/file_index/bitmap/bitmap_file_index_meta.cpp
+++ b/src/paimon/common/file_index/bitmap/bitmap_file_index_meta.cpp
@@ -18,11 +18,13 @@
#include "paimon/common/file_index/bitmap/bitmap_file_index_meta.h"
+#include <cstdint>
#include <string>
#include <utility>
#include "fmt/format.h"
#include "paimon/common/utils/field_type_utils.h"
+#include "paimon/common/utils/math.h"
#include "paimon/defs.h"
#include "paimon/io/data_input_stream.h"
#include "paimon/memory/bytes.h"
@@ -80,6 +82,18 @@ Result<std::function<void(const Literal&)>>
BitmapFileIndexMeta::GetValueWriter(
[output_stream](const Literal& literal) -> void {
output_stream->WriteValue<int64_t>(literal.GetValue<int64_t>());
});
+ case FieldType::FLOAT:
+ return std::function<void(const Literal&)>(
+ [output_stream](const Literal& literal) -> void {
+ const float value =
CanonicalizeFloatingPoint(literal.GetValue<float>());
+ output_stream->WriteValue<float>(value);
+ });
+ case FieldType::DOUBLE:
+ return std::function<void(const Literal&)>(
+ [output_stream](const Literal& literal) -> void {
+ const double value =
CanonicalizeFloatingPoint(literal.GetValue<double>());
+ output_stream->WriteValue<double>(value);
+ });
case FieldType::STRING:
return std::function<void(const Literal&)>(
[output_stream](const Literal& literal) -> void {
@@ -155,6 +169,24 @@ Result<std::function<Result<Literal>()>>
BitmapFileIndexMeta::GetValueReader(
};
return func;
}
+ case FieldType::FLOAT: {
+ std::function<Result<Literal>()> func = [&in, move_body_start,
+ this]() ->
Result<Literal> {
+ PAIMON_ASSIGN_OR_RAISE(float value,
+ ReadAndMoveBodyStart<float>(in,
move_body_start));
+ return Literal(value);
+ };
+ return func;
+ }
+ case FieldType::DOUBLE: {
+ std::function<Result<Literal>()> func = [&in, move_body_start,
+ this]() ->
Result<Literal> {
+ PAIMON_ASSIGN_OR_RAISE(double value,
+ ReadAndMoveBodyStart<double>(in,
move_body_start));
+ return Literal(value);
+ };
+ return func;
+ }
case FieldType::DATE: {
std::function<Result<Literal>()> func = [&in, move_body_start,
this]() ->
Result<Literal> {
diff --git a/src/paimon/common/file_index/bitmap/bitmap_file_index_test.cpp
b/src/paimon/common/file_index/bitmap/bitmap_file_index_test.cpp
index 42a888a8..e8f74c83 100644
--- a/src/paimon/common/file_index/bitmap/bitmap_file_index_test.cpp
+++ b/src/paimon/common/file_index/bitmap/bitmap_file_index_test.cpp
@@ -27,6 +27,7 @@
#include "paimon/common/utils/arrow/status_utils.h"
#include "paimon/common/utils/checked_cast.h"
#include "paimon/common/utils/date_time_utils.h"
+#include "paimon/common/utils/math.h"
#include "paimon/data/timestamp.h"
#include "paimon/defs.h"
#include "paimon/file_index/bitmap_index_result.h"
@@ -36,6 +37,15 @@
#include "paimon/memory/memory_pool.h"
#include "paimon/testing/utils/testharness.h"
namespace paimon::test {
+namespace {
+
+template <size_t N>
+std::vector<char> JavaBytes(const char (&bytes)[N]) {
+ return std::vector<char>(bytes, bytes + N - 1);
+}
+
+} // namespace
+
class BitmapIndexTest : public ::testing::Test {
public:
void SetUp() override {
@@ -78,6 +88,21 @@ class BitmapIndexTest : public ::testing::Test {
return writer->SerializedBytes();
}
+ template <typename ArrowBuilder, typename ValueType>
+ Result<std::shared_ptr<arrow::Array>> CreateArray(const
std::shared_ptr<arrow::DataType>& type,
+ const
std::vector<ValueType>& values) const {
+ auto value_builder = std::make_shared<ArrowBuilder>();
+ for (ValueType value : values) {
+ PAIMON_RETURN_NOT_OK_FROM_ARROW(value_builder->Append(value));
+ }
+ PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Array>
value_array,
+ value_builder->Finish());
+ PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+ std::shared_ptr<arrow::StructArray> struct_array,
+ arrow::StructArray::Make({value_array}, {arrow::field("f0",
type)}));
+ return struct_array;
+ }
+
private:
std::shared_ptr<MemoryPool> pool_;
};
@@ -620,6 +645,226 @@ TEST_F(BitmapIndexTest, TestTimestampType) {
}
}
+TEST_F(BitmapIndexTest, TestFloatAndDoubleTypes) {
+ const auto check_float = [&](int32_t version) {
+ const auto type = arrow::float32();
+ auto array =
+
arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({arrow::field("f0",
type)}),
+ R"([[1.25], [null],
[-2.5], [1.25], [3.75]])")
+ .ValueOrDie();
+ ASSERT_OK_AND_ASSIGN(PAIMON_UNIQUE_PTR<Bytes> index_bytes,
+ WriteIndex(type, version, array));
+ auto input_stream =
+ std::make_shared<ByteArrayInputStream>(index_bytes->data(),
index_bytes->size());
+ BitmapFileIndex file_index({});
+ ASSERT_OK_AND_ASSIGN(auto reader,
+
file_index.CreateReader(CreateArrowSchema(type).get(), 0,
+ index_bytes->size(),
input_stream, pool_));
+ CheckResult(reader->VisitEqual(Literal(1.25f)).value(), {0, 3});
+ CheckResult(reader->VisitNotEqual(Literal(1.25f)).value(), {2, 4});
+ CheckResult(reader->VisitIsNull().value(), {1});
+ };
+
+ const auto check_double = [&](int32_t version) {
+ const auto type = arrow::float64();
+ auto array =
+
arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({arrow::field("f0",
type)}),
+ R"([[1.25], [null],
[-2.5], [1.25], [3.75]])")
+ .ValueOrDie();
+ ASSERT_OK_AND_ASSIGN(PAIMON_UNIQUE_PTR<Bytes> index_bytes,
+ WriteIndex(type, version, array));
+ auto input_stream =
+ std::make_shared<ByteArrayInputStream>(index_bytes->data(),
index_bytes->size());
+ BitmapFileIndex file_index({});
+ ASSERT_OK_AND_ASSIGN(auto reader,
+
file_index.CreateReader(CreateArrowSchema(type).get(), 0,
+ index_bytes->size(),
input_stream, pool_));
+ CheckResult(reader->VisitEqual(Literal(1.25)).value(), {0, 3});
+ CheckResult(reader->VisitNotEqual(Literal(1.25)).value(), {2, 4});
+ CheckResult(reader->VisitIsNull().value(), {1});
+ };
+
+ for (int32_t version : {1, 2}) {
+ check_float(version);
+ check_double(version);
+ }
+}
+
+TEST_F(BitmapIndexTest, TestFloatingPointJavaCompatibility) {
+ const std::vector<char> java_float_v1 = JavaBytes(
+ "\x01\x00\x00\x00\x08\x00\x00\x00\x03\x00\x00\x00\x00\x00\x00\x00"
+ "\x00\x00\x80\x00\x00\x00\x00\x00\x00\x14\x7f\xc0\x00\x00\x00\x00"
+ "\x00\x28\x3a\x30\x00\x00\x01\x00\x00\x00\x00\x00\x01\x00\x10\x00"
+ "\x00\x00\x04\x00\x07\x00\x3a\x30\x00\x00\x01\x00\x00\x00\x00\x00"
+ "\x01\x00\x10\x00\x00\x00\x03\x00\x06\x00\x3a\x30\x00\x00\x01\x00"
+ "\x00\x00\x00\x00\x03\x00\x10\x00\x00\x00\x00\x00\x01\x00\x02\x00"
+ "\x05\x00");
+ const std::vector<char> java_float_v2 = JavaBytes(
+ "\x02\x00\x00\x00\x08\x00\x00\x00\x03\x00\x00\x00\x00\x01\x80\x00"
+ "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x28\x00\x00\x00\x03\x80\x00"
+ "\x00\x00\x00\x00\x00\x14\x00\x00\x00\x14\x00\x00\x00\x00\x00\x00"
+ "\x00\x00\x00\x00\x00\x14\x7f\xc0\x00\x00\x00\x00\x00\x28\x00\x00"
+ "\x00\x18\x3a\x30\x00\x00\x01\x00\x00\x00\x00\x00\x01\x00\x10\x00"
+ "\x00\x00\x04\x00\x07\x00\x3a\x30\x00\x00\x01\x00\x00\x00\x00\x00"
+ "\x01\x00\x10\x00\x00\x00\x03\x00\x06\x00\x3a\x30\x00\x00\x01\x00"
+ "\x00\x00\x00\x00\x03\x00\x10\x00\x00\x00\x00\x00\x01\x00\x02\x00"
+ "\x05\x00");
+ const std::vector<char> java_double_v1 = JavaBytes(
+ "\x01\x00\x00\x00\x08\x00\x00\x00\x03\x00\x00\x00\x00\x00\x00\x00"
+ "\x00\x00\x00\x00\x00\x00\x80\x00\x00\x00\x00\x00\x00\x00\x00\x00"
+ "\x00\x14\x7f\xf8\x00\x00\x00\x00\x00\x00\x00\x00\x00\x28\x3a\x30"
+ "\x00\x00\x01\x00\x00\x00\x00\x00\x01\x00\x10\x00\x00\x00\x04\x00"
+ "\x07\x00\x3a\x30\x00\x00\x01\x00\x00\x00\x00\x00\x01\x00\x10\x00"
+ "\x00\x00\x03\x00\x06\x00\x3a\x30\x00\x00\x01\x00\x00\x00\x00\x00"
+ "\x03\x00\x10\x00\x00\x00\x00\x00\x01\x00\x02\x00\x05\x00");
+ const std::vector<char> java_double_v2 = JavaBytes(
+ "\x02\x00\x00\x00\x08\x00\x00\x00\x03\x00\x00\x00\x00\x01\x80\x00"
+ "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x34\x00\x00"
+ "\x00\x03\x80\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x14\x00\x00"
+ "\x00\x14\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"
+ "\x00\x14\x7f\xf8\x00\x00\x00\x00\x00\x00\x00\x00\x00\x28\x00\x00"
+ "\x00\x18\x3a\x30\x00\x00\x01\x00\x00\x00\x00\x00\x01\x00\x10\x00"
+ "\x00\x00\x04\x00\x07\x00\x3a\x30\x00\x00\x01\x00\x00\x00\x00\x00"
+ "\x01\x00\x10\x00\x00\x00\x03\x00\x06\x00\x3a\x30\x00\x00\x01\x00"
+ "\x00\x00\x00\x00\x03\x00\x10\x00\x00\x00\x00\x00\x01\x00\x02\x00"
+ "\x05\x00");
+ const std::vector<char> java_float_nan_v1 = JavaBytes(
+ "\x01\x00\x00\x00\x03\x00\x00\x00\x01\x00\x7f\xc0\x00\x00\x00\x00"
+ "\x00\x00\x3a\x30\x00\x00\x01\x00\x00\x00\x00\x00\x02\x00\x10\x00"
+ "\x00\x00\x00\x00\x01\x00\x02\x00");
+ const std::vector<char> java_float_nan_v2 = JavaBytes(
+ "\x02\x00\x00\x00\x03\x00\x00\x00\x01\x00\x00\x00\x00\x01\x7f\xc0"
+ "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x00\x00\x00\x01\x7f\xc0"
+ "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x16\x3a\x30\x00\x00\x01\x00"
+ "\x00\x00\x00\x00\x02\x00\x10\x00\x00\x00\x00\x00\x01\x00\x02\x00");
+ const std::vector<char> java_double_nan_v1 = JavaBytes(
+ "\x01\x00\x00\x00\x03\x00\x00\x00\x01\x00\x7f\xf8\x00\x00\x00\x00"
+ "\x00\x00\x00\x00\x00\x00\x3a\x30\x00\x00\x01\x00\x00\x00\x00\x00"
+ "\x02\x00\x10\x00\x00\x00\x00\x00\x01\x00\x02\x00");
+ const std::vector<char> java_double_nan_v2 = JavaBytes(
+ "\x02\x00\x00\x00\x03\x00\x00\x00\x01\x00\x00\x00\x00\x01\x7f\xf8"
+ "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x14\x00\x00"
+ "\x00\x01\x7f\xf8\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"
+ "\x00\x16\x3a\x30\x00\x00\x01\x00\x00\x00\x00\x00\x02\x00\x10\x00"
+ "\x00\x00\x00\x00\x01\x00\x02\x00");
+
+ const auto float_nan =
FloatingPointFromBits<float>(kCanonicalFloatNaNBits);
+ const auto float_positive_payload_nan =
FloatingPointFromBits<float>(uint32_t{0x7fc12345});
+ const auto float_negative_payload_nan =
FloatingPointFromBits<float>(uint32_t{0xffc54321});
+ const std::vector<float> float_values = {float_nan,
+ float_positive_payload_nan,
+ float_negative_payload_nan,
+ -0.0f,
+ +0.0f,
+ float_negative_payload_nan,
+ -0.0f,
+ +0.0f};
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::Array> float_array,
+ (CreateArray<arrow::FloatBuilder>(arrow::float32(),
float_values)));
+
+ const auto double_nan =
FloatingPointFromBits<double>(kCanonicalDoubleNaNBits);
+ const auto double_positive_payload_nan =
+ FloatingPointFromBits<double>(uint64_t{0x7ff8123456789abc});
+ const auto double_negative_payload_nan =
+ FloatingPointFromBits<double>(uint64_t{0xfff8abcdef012345});
+ const std::vector<double> double_values = {double_nan,
+ double_positive_payload_nan,
+ double_negative_payload_nan,
+ -0.0,
+ +0.0,
+ double_negative_payload_nan,
+ -0.0,
+ +0.0};
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::Array> double_array,
+ (CreateArray<arrow::DoubleBuilder>(arrow::float64(),
double_values)));
+
+ const auto check_reader = [&](const std::shared_ptr<FileIndexReader>&
reader,
+ const std::vector<Literal>& nan_literals,
+ const Literal& negative_zero, const Literal&
positive_zero) {
+ for (const Literal& nan_literal : nan_literals) {
+ CheckResult(reader->VisitEqual(nan_literal).value(), {0, 1, 2, 5});
+ }
+ CheckResult(reader->VisitEqual(negative_zero).value(), {3, 6});
+ CheckResult(reader->VisitEqual(positive_zero).value(), {4, 7});
+ };
+
+ const auto check = [&](const std::shared_ptr<arrow::DataType>& type,
+ const std::shared_ptr<arrow::Array>& array,
+ const std::vector<Literal>& nan_literals, const
Literal& negative_zero,
+ const Literal& positive_zero, int32_t version,
+ const std::vector<char>& java_bytes) {
+ auto input_stream =
+ std::make_shared<ByteArrayInputStream>(java_bytes.data(),
java_bytes.size());
+ BitmapFileIndex file_index({});
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<FileIndexReader> reader,
+
file_index.CreateReader(CreateArrowSchema(type).get(), 0,
+ java_bytes.size(),
input_stream, pool_));
+ check_reader(reader, nan_literals, negative_zero, positive_zero);
+
+ ASSERT_OK_AND_ASSIGN(PAIMON_UNIQUE_PTR<Bytes> cpp_bytes,
WriteIndex(type, version, array));
+ auto cpp_input_stream =
+ std::make_shared<ByteArrayInputStream>(cpp_bytes->data(),
cpp_bytes->size());
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<FileIndexReader> cpp_reader,
+
file_index.CreateReader(CreateArrowSchema(type).get(), 0,
+ cpp_bytes->size(),
cpp_input_stream, pool_));
+ check_reader(cpp_reader, nan_literals, negative_zero, positive_zero);
+ };
+
+ const auto check_nan_meta = [&](const std::shared_ptr<arrow::DataType>&
type,
+ const std::shared_ptr<arrow::Array>&
array, int32_t version,
+ size_t key_size, const std::vector<char>&
java_bytes) {
+ ASSERT_OK_AND_ASSIGN(PAIMON_UNIQUE_PTR<Bytes> cpp_bytes,
WriteIndex(type, version, array));
+ // RoaringBitmap may choose a different, semantically equivalent body
encoding. Compare
+ // only the prefix which is independent of that encoding. For V1 it
contains version,
+ // row count, entry count, has-null flag, key and offset: 1 + 4 + 4 +
1 + key_size + 4.
+ // For V2 it additionally contains one secondary-index entry,
bitmap-body offset, and one
+ // index-block entry through entry.offset, but excludes entry.length
and the bitmap body:
+ // 1 + 4 + 4 + 1 + 4 + (key_size + 4) + 4 + 4 + (key_size + 4).
+ const size_t comparable_prefix_size =
+ version == BitmapFileIndex::VERSION_1 ? 14 + key_size : 30 + 2 *
key_size;
+ ASSERT_GE(java_bytes.size(), comparable_prefix_size);
+ ASSERT_GE(cpp_bytes->size(), comparable_prefix_size);
+ ASSERT_EQ(
+ std::vector<char>(java_bytes.begin(), java_bytes.begin() +
comparable_prefix_size),
+ std::vector<char>(cpp_bytes->data(), cpp_bytes->data() +
comparable_prefix_size));
+ };
+
+ check(arrow::float32(), float_array,
+ {Literal(float_nan), Literal(float_positive_payload_nan),
+ Literal(float_negative_payload_nan)},
+ Literal(-0.0f), Literal(+0.0f), /*version=*/1, java_float_v1);
+ check(arrow::float32(), float_array,
+ {Literal(float_nan), Literal(float_positive_payload_nan),
+ Literal(float_negative_payload_nan)},
+ Literal(-0.0f), Literal(+0.0f), /*version=*/2, java_float_v2);
+ check(arrow::float64(), double_array,
+ {Literal(double_nan), Literal(double_positive_payload_nan),
+ Literal(double_negative_payload_nan)},
+ Literal(-0.0), Literal(+0.0), /*version=*/1, java_double_v1);
+ check(arrow::float64(), double_array,
+ {Literal(double_nan), Literal(double_positive_payload_nan),
+ Literal(double_negative_payload_nan)},
+ Literal(-0.0), Literal(+0.0), /*version=*/2, java_double_v2);
+
+ const std::vector<float> float_nan_values = {float_negative_payload_nan,
+ float_positive_payload_nan,
float_nan};
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::Array> float_nan_array,
+ (CreateArray<arrow::FloatBuilder>(arrow::float32(),
float_nan_values)));
+ check_nan_meta(arrow::float32(), float_nan_array, /*version=*/1,
sizeof(float),
+ java_float_nan_v1);
+ check_nan_meta(arrow::float32(), float_nan_array, /*version=*/2,
sizeof(float),
+ java_float_nan_v2);
+
+ const std::vector<double> double_nan_values = {double_negative_payload_nan,
+
double_positive_payload_nan, double_nan};
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::Array> double_nan_array,
+ (CreateArray<arrow::DoubleBuilder>(arrow::float64(),
double_nan_values)));
+ check_nan_meta(arrow::float64(), double_nan_array, /*version=*/1,
sizeof(double),
+ java_double_nan_v1);
+ check_nan_meta(arrow::float64(), double_nan_array, /*version=*/2,
sizeof(double),
+ java_double_nan_v2);
+}
+
TEST_F(BitmapIndexTest, TestHighCardinalityForCompatibility) {
auto type = arrow::utf8();
auto check_result = [&](const std::string& index_file_name) {
diff --git
a/src/paimon/common/file_index/bsi/bit_slice_index_bitmap_file_index.cpp
b/src/paimon/common/file_index/bsi/bit_slice_index_bitmap_file_index.cpp
index 4cb98cdc..558dec50 100644
--- a/src/paimon/common/file_index/bsi/bit_slice_index_bitmap_file_index.cpp
+++ b/src/paimon/common/file_index/bsi/bit_slice_index_bitmap_file_index.cpp
@@ -38,6 +38,7 @@
#include "paimon/common/utils/checked_cast.h"
#include "paimon/common/utils/date_time_utils.h"
#include "paimon/common/utils/field_type_utils.h"
+#include "paimon/data/decimal.h"
#include "paimon/data/timestamp.h"
#include "paimon/defs.h"
#include "paimon/file_index/bitmap_index_result.h"
@@ -252,10 +253,26 @@ Result<BitSliceIndexBitmapFileIndex::ValueMapperType>
BitSliceIndexBitmapFileInd
return literal.GetValue<Timestamp>().ToMicrosecond();
});
}
+ case FieldType::DECIMAL:
+ return BitSliceIndexBitmapFileIndex::ValueMapperType(
+ [](const Literal& literal) -> Result<int64_t> {
+ if (literal.IsNull()) {
+ return Status::Invalid(
+ "literal cannot be null when GetValue in
BitSliceIndexBitmapFileIndex");
+ }
+ const auto value = literal.GetValue<Decimal>();
+ if (value.Value() < std::numeric_limits<int64_t>::min() ||
+ value.Value() > std::numeric_limits<int64_t>::max()) {
+ return Status::Invalid(fmt::format(
+ "decimal unscaled value {} does not fit in int64
for bsi index",
+ value.ToString()));
+ }
+ return value.ToUnscaledLong();
+ });
default:
- // TODO(xinyu.lxy): support decimal
return Status::Invalid(
- "BitSliceIndexBitmapFileIndex only support
TINYINT/SMALLINT/INT/BIGINT/DATE");
+ "BitSliceIndexBitmapFileIndex only support "
+ "TINYINT/SMALLINT/INT/BIGINT/DATE/TIMESTAMP/DECIMAL");
}
}
diff --git
a/src/paimon/common/file_index/bsi/bit_slice_index_bitmap_file_index_test.cpp
b/src/paimon/common/file_index/bsi/bit_slice_index_bitmap_file_index_test.cpp
index 760434e1..8da7ace2 100644
---
a/src/paimon/common/file_index/bsi/bit_slice_index_bitmap_file_index_test.cpp
+++
b/src/paimon/common/file_index/bsi/bit_slice_index_bitmap_file_index_test.cpp
@@ -27,6 +27,7 @@
#include "gtest/gtest.h"
#include "paimon/common/utils/arrow/status_utils.h"
#include "paimon/common/utils/field_type_utils.h"
+#include "paimon/data/decimal.h"
#include "paimon/data/timestamp.h"
#include "paimon/defs.h"
#include "paimon/file_index/bitmap_index_result.h"
@@ -420,6 +421,31 @@ TEST_F(BitSliceIndexBitmapIndexReaderTest,
TestTimestampType) {
"literal cannot be null when GetValue in
BitSliceIndexBitmapFileIndex");
}
+TEST_F(BitSliceIndexBitmapIndexReaderTest, TestDecimalType) {
+ const auto type = arrow::decimal128(10, 2);
+ ASSERT_OK_AND_ASSIGN(PAIMON_UNIQUE_PTR<Bytes> index_bytes,
+ WriteIndex(type, R"([["1.00"], ["2.50"], [null],
["-1.25"], ["2.50"]])"));
+ auto input_stream =
+ std::make_shared<ByteArrayInputStream>(index_bytes->data(),
index_bytes->size());
+ BitSliceIndexBitmapFileIndex file_index({});
+ ASSERT_OK_AND_ASSIGN(
+ auto reader, file_index.CreateReader(CreateArrowSchema(type).get(), 0,
index_bytes->size(),
+ input_stream, pool_));
+
+ CheckResult(reader->VisitEqual(Literal(Decimal(10, 2, 250))).value(), {1,
4});
+ CheckResult(reader->VisitGreaterThan(Literal(Decimal(10, 2,
100))).value(), {1, 4});
+ CheckResult(reader->VisitLessThan(Literal(Decimal(10, 2, 0))).value(),
{3});
+ CheckResult(reader->VisitIsNull().value(), {2});
+
+ // BSI does not rescale Decimal literals. A mathematically equivalent
literal with a
+ // different scale produces an incorrect empty result, so callers must use
the field's scale.
+ CheckResult(reader->VisitEqual(Literal(Decimal(10, 3, 2500))).value(), {});
+
+ // test invalid case for decimal128(20, 0) which exceeds int64 range
+ ASSERT_NOK_WITH_MSG(WriteIndex(arrow::decimal128(20, 0),
R"([["9223372036854775808"]])"),
+ "does not fit in int64 for bsi index");
+}
+
TEST_F(BitSliceIndexBitmapIndexReaderTest, TestUnInvalidType) {
std::vector<char> index_bytes = {
1, 0, 0, 0, 5, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 2, 58,
diff --git
a/src/paimon/common/file_index/rangebitmap/dictionary/chunked_dictionary.cpp
b/src/paimon/common/file_index/rangebitmap/dictionary/chunked_dictionary.cpp
index 3011afea..cc96e48f 100644
--- a/src/paimon/common/file_index/rangebitmap/dictionary/chunked_dictionary.cpp
+++ b/src/paimon/common/file_index/rangebitmap/dictionary/chunked_dictionary.cpp
@@ -89,14 +89,11 @@ Result<std::shared_ptr<Chunk>>
ChunkedDictionary::GetChunk(int32_t index) {
if (index < 0 || index >= size_) {
return Status::Invalid(fmt::format("Invalid chunk index: {}", index));
}
- if (offsets_bytes_ == nullptr || chunks_bytes_ == nullptr) {
+ if (offsets_bytes_ == nullptr) {
PAIMON_RETURN_NOT_OK(input_stream_->Seek(body_offset_, FS_SEEK_SET));
auto offsets = Bytes::AllocateBytes(offsets_length_, pool_.get());
PAIMON_RETURN_NOT_OK(input_stream_->Read(offsets->data(),
offsets_length_));
offsets_bytes_ = std::move(offsets);
- auto chunks = Bytes::AllocateBytes(chunks_length_, pool_.get());
- PAIMON_RETURN_NOT_OK(input_stream_->Read(chunks->data(),
chunks_length_));
- chunks_bytes_ = std::move(chunks);
}
if (chunks_cache_[index]) {
return chunks_cache_[index];
@@ -244,6 +241,5 @@ ChunkedDictionary::ChunkedDictionary(const
std::shared_ptr<InputStream>& input_s
chunks_length_(chunks_length),
body_offset_(body_offset),
offsets_bytes_(nullptr),
- chunks_bytes_(nullptr),
chunks_cache_(std::vector<std::shared_ptr<Chunk>>(size)) {}
} // namespace paimon
diff --git
a/src/paimon/common/file_index/rangebitmap/dictionary/chunked_dictionary.h
b/src/paimon/common/file_index/rangebitmap/dictionary/chunked_dictionary.h
index 7cb4ba4f..a1a6edcf 100644
--- a/src/paimon/common/file_index/rangebitmap/dictionary/chunked_dictionary.h
+++ b/src/paimon/common/file_index/rangebitmap/dictionary/chunked_dictionary.h
@@ -96,7 +96,6 @@ class ChunkedDictionary final : public Dictionary {
// for lazy loading
PAIMON_UNIQUE_PTR<Bytes> offsets_bytes_;
- PAIMON_UNIQUE_PTR<Bytes> chunks_bytes_;
// mmap chunks cache
std::vector<std::shared_ptr<Chunk>> chunks_cache_;
diff --git
a/src/paimon/common/file_index/rangebitmap/dictionary/chunked_dictionary_test.cpp
b/src/paimon/common/file_index/rangebitmap/dictionary/chunked_dictionary_test.cpp
index 93c19854..671d891c 100644
---
a/src/paimon/common/file_index/rangebitmap/dictionary/chunked_dictionary_test.cpp
+++
b/src/paimon/common/file_index/rangebitmap/dictionary/chunked_dictionary_test.cpp
@@ -456,9 +456,25 @@ TEST_F(ChunkedDictionaryTest,
TestKeyFactoryUnsupportedType) {
"Unsupported field type for KeyFactory: BINARY");
}
-TEST_F(ChunkedDictionaryTest, TestStringKeyFactoryNotImplemented) {
- ASSERT_NOK_WITH_MSG(KeyFactory::Create(FieldType::STRING),
- "Unsupported field type for KeyFactory: STRING");
+TEST_F(ChunkedDictionaryTest, TestStringKeyFactory) {
+ ASSERT_OK_AND_ASSIGN(auto key_factory,
KeyFactory::Create(FieldType::STRING));
+ ASSERT_OK_AND_ASSIGN(auto appender,
+ ChunkedDictionary::Appender::Create(key_factory, 12,
pool_));
+ ASSERT_OK(appender->AppendSorted(Literal(FieldType::STRING, "apple", 5),
0));
+ ASSERT_OK(appender->AppendSorted(Literal(FieldType::STRING, "banana", 6),
1));
+ ASSERT_OK(appender->AppendSorted(Literal(FieldType::STRING, "pear", 4),
2));
+ ASSERT_OK_AND_ASSIGN(auto bytes, appender->Serialize());
+ auto input_stream = std::make_shared<ByteArrayInputStream>(bytes->data(),
bytes->size());
+ ASSERT_OK_AND_ASSIGN(auto dict,
+ ChunkedDictionary::Create(FieldType::STRING,
input_stream, 0, pool_));
+
+ ASSERT_OK_AND_ASSIGN(int32_t banana_code,
dict->Find(Literal(FieldType::STRING, "banana", 6)));
+ ASSERT_EQ(banana_code, 1);
+ ASSERT_OK_AND_ASSIGN(Literal pear, dict->Find(2));
+ ASSERT_EQ(pear.GetValue<std::string>(), "pear");
+ ASSERT_OK_AND_ASSIGN(int32_t between_code,
+ dict->Find(Literal(FieldType::STRING, "blueberry",
9)));
+ ASSERT_EQ(between_code, -3);
}
TEST_F(ChunkedDictionaryTest, TestFindByCodeInvalidNegative) {
diff --git
a/src/paimon/common/file_index/rangebitmap/dictionary/key_factory.cpp
b/src/paimon/common/file_index/rangebitmap/dictionary/key_factory.cpp
index 37f80c31..a3fafd59 100644
--- a/src/paimon/common/file_index/rangebitmap/dictionary/key_factory.cpp
+++ b/src/paimon/common/file_index/rangebitmap/dictionary/key_factory.cpp
@@ -23,6 +23,7 @@
#include "fmt/format.h"
#include "paimon/common/file_index/rangebitmap/dictionary/chunked_dictionary.h"
#include "paimon/common/file_index/rangebitmap/dictionary/fixed_length_chunk.h"
+#include
"paimon/common/file_index/rangebitmap/dictionary/variable_length_chunk.h"
#include
"paimon/common/file_index/rangebitmap/utils/literal_serialization_utils.h"
#include "paimon/common/utils/field_type_utils.h"
#include "paimon/common/utils/fields_comparator.h"
@@ -48,6 +49,8 @@ Result<std::shared_ptr<KeyFactory>>
KeyFactory::Create(FieldType field_type) {
return std::make_shared<FloatKeyFactory>();
case FieldType::DOUBLE:
return std::make_shared<DoubleKeyFactory>();
+ case FieldType::STRING:
+ return std::make_shared<StringKeyFactory>();
default:
return Status::Invalid(fmt::format("Unsupported field type for
KeyFactory: {}",
FieldTypeUtils::FieldTypeToString(field_type)));
@@ -91,12 +94,31 @@ Result<std::unique_ptr<Chunk>>
FixedLengthKeyFactory::MmapChunk(
Result<std::unique_ptr<Chunk>> VariableLengthKeyFactory::CreateChunk(
const Literal& key, int32_t code, int32_t keys_length_limit,
const std::shared_ptr<MemoryPool>& pool) {
- return Status::NotImplemented("VariableLengthKeyFactory::CreateChunk not
implemented");
+ PAIMON_ASSIGN_OR_RAISE(LiteralSerDeUtils::Serializer serializer,
+
LiteralSerDeUtils::CreateValueWriter(GetFieldType()));
+ return std::make_unique<VariableLengthChunk>(key, code, keys_length_limit,
shared_from_this(),
+ serializer, pool);
}
Result<std::unique_ptr<Chunk>> VariableLengthKeyFactory::MmapChunk(
const std::shared_ptr<InputStream>& input_stream, int32_t chunk_offset,
int32_t keys_base_offset, const std::shared_ptr<MemoryPool>& pool) {
- return Status::NotImplemented("VariableLengthKeyFactory::MmapChunk not
implemented");
+ PAIMON_RETURN_NOT_OK(input_stream->Seek(chunk_offset, FS_SEEK_SET));
+ const auto data_in = std::make_shared<DataInputStream>(input_stream);
+ PAIMON_ASSIGN_OR_RAISE(int8_t version, data_in->ReadValue<int8_t>());
+ if (version != VariableLengthChunk::kCurrentVersion) {
+ return Status::Invalid(fmt::format("Unsupported version for
KeyFactory: {}", version));
+ }
+ PAIMON_ASSIGN_OR_RAISE(LiteralSerDeUtils::Deserializer deserializer,
+
LiteralSerDeUtils::CreateValueReader(GetFieldType()));
+ PAIMON_ASSIGN_OR_RAISE(Literal key_literal, deserializer(data_in,
pool.get()));
+ PAIMON_ASSIGN_OR_RAISE(int32_t code, data_in->ReadValue<int32_t>());
+ PAIMON_ASSIGN_OR_RAISE(int32_t offset, data_in->ReadValue<int32_t>());
+ PAIMON_ASSIGN_OR_RAISE(int32_t size, data_in->ReadValue<int32_t>());
+ PAIMON_ASSIGN_OR_RAISE(int32_t offsets_length,
data_in->ReadValue<int32_t>());
+ PAIMON_ASSIGN_OR_RAISE(int32_t keys_length, data_in->ReadValue<int32_t>());
+ return std::make_unique<VariableLengthChunk>(key_literal, code, offset,
size,
+ shared_from_this(),
input_stream, keys_base_offset,
+ offsets_length, keys_length,
pool);
}
/// Java-compatible ordering for floats
diff --git
a/src/paimon/common/file_index/rangebitmap/dictionary/variable_length_chunk.cpp
b/src/paimon/common/file_index/rangebitmap/dictionary/variable_length_chunk.cpp
new file mode 100644
index 00000000..5bb6dda4
--- /dev/null
+++
b/src/paimon/common/file_index/rangebitmap/dictionary/variable_length_chunk.cpp
@@ -0,0 +1,171 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include
"paimon/common/file_index/rangebitmap/dictionary/variable_length_chunk.h"
+
+#include <limits>
+#include <utility>
+
+#include "fmt/format.h"
+#include "paimon/common/file_index/rangebitmap/dictionary/key_factory.h"
+#include "paimon/common/io/memory_segment_output_stream.h"
+#include "paimon/common/memory/memory_segment_utils.h"
+#include "paimon/io/byte_array_input_stream.h"
+#include "paimon/io/data_input_stream.h"
+#include "paimon/memory/bytes.h"
+
+namespace paimon {
+
+Result<bool> VariableLengthChunk::TryAdd(const Literal& key) {
+ PAIMON_ASSIGN_OR_RAISE(int32_t key_length,
LiteralSerDeUtils::GetSerializedSizeInBytes(key));
+ if (key_length > remaining_keys_size_ ||
+ static_cast<int32_t>(sizeof(int32_t)) > remaining_offsets_size_) {
+ return false;
+ }
+
offsets_stream_out_->WriteValue<int32_t>(static_cast<int32_t>(keys_stream_out_->CurrentSize()));
+ PAIMON_RETURN_NOT_OK(serializer_(keys_stream_out_, key));
+ remaining_offsets_size_ -= sizeof(int32_t);
+ remaining_keys_size_ -= key_length;
+ ++size_;
+ return true;
+}
+
+Result<int32_t> VariableLengthChunk::CompareKey(const Literal& lhs, const
Literal& rhs) {
+ return factory_->CompareLiteral(lhs, rhs);
+}
+
+Status VariableLengthChunk::LoadKeys() {
+ if (offsets_ != nullptr && keys_ != nullptr) {
+ return Status::OK();
+ }
+ if (offsets_length_ < 0 || keys_length_ < 0 ||
+ offsets_length_ > std::numeric_limits<int32_t>::max() - keys_length_) {
+ return Status::Invalid("Invalid variable length chunk payload length");
+ }
+ PAIMON_RETURN_NOT_OK(input_stream_->Seek(keys_base_offset_ + offset_,
FS_SEEK_SET));
+ offsets_ = Bytes::AllocateBytes(offsets_length_, pool_.get());
+ PAIMON_ASSIGN_OR_RAISE(int64_t offsets_read,
+ input_stream_->Read(offsets_->data(),
offsets_length_));
+ if (offsets_read != offsets_length_) {
+ return Status::Invalid(fmt::format(
+ "Failed to read variable length chunk offsets, expected {} bytes
but got {}",
+ offsets_length_, offsets_read));
+ }
+ keys_ = Bytes::AllocateBytes(keys_length_, pool_.get());
+ PAIMON_ASSIGN_OR_RAISE(int64_t keys_read,
input_stream_->Read(keys_->data(), keys_length_));
+ if (keys_read != keys_length_) {
+ return Status::Invalid(
+ fmt::format("Failed to read variable length chunk keys, expected
{} bytes but got {}",
+ keys_length_, keys_read));
+ }
+ PAIMON_ASSIGN_OR_RAISE(deserializer_,
+
LiteralSerDeUtils::CreateValueReader(factory_->GetFieldType()));
+ return Status::OK();
+}
+
+Result<Literal> VariableLengthChunk::GetKey(int32_t index) {
+ if (index < 0 || index >= size_) {
+ return Status::Invalid("Index out of bounds");
+ }
+ PAIMON_RETURN_NOT_OK(LoadKeys());
+ auto offsets_in = std::make_shared<DataInputStream>(
+ std::make_shared<ByteArrayInputStream>(offsets_->data(),
offsets_->size()));
+ PAIMON_RETURN_NOT_OK(offsets_in->Seek(static_cast<int64_t>(index) *
sizeof(int32_t)));
+ PAIMON_ASSIGN_OR_RAISE(int32_t key_offset,
offsets_in->ReadValue<int32_t>());
+ if (key_offset < 0 || key_offset >= keys_length_) {
+ return Status::Invalid("Invalid key offset in variable length chunk");
+ }
+ auto keys_in = std::make_shared<DataInputStream>(
+ std::make_shared<ByteArrayInputStream>(keys_->data(), keys_->size()));
+ PAIMON_RETURN_NOT_OK(keys_in->Seek(key_offset));
+ return deserializer_(keys_in, pool_.get());
+}
+
+Result<PAIMON_UNIQUE_PTR<Bytes>> VariableLengthChunk::SerializeChunk() const {
+ const auto data_out = std::make_shared<MemorySegmentOutputStream>(
+ MemorySegmentOutputStream::DEFAULT_SEGMENT_SIZE, pool_);
+ data_out->WriteValue<int8_t>(kCurrentVersion);
+ PAIMON_RETURN_NOT_OK(serializer_(data_out, key_));
+ data_out->WriteValue<int32_t>(code_);
+ data_out->WriteValue<int32_t>(offset_);
+ data_out->WriteValue<int32_t>(size_);
+
data_out->WriteValue<int32_t>(static_cast<int32_t>(offsets_stream_out_->CurrentSize()));
+
data_out->WriteValue<int32_t>(static_cast<int32_t>(keys_stream_out_->CurrentSize()));
+ return MemorySegmentUtils::CopyToBytes(
+ data_out->Segments(), 0,
static_cast<int32_t>(data_out->CurrentSize()), pool_.get());
+}
+
+Result<PAIMON_UNIQUE_PTR<Bytes>> VariableLengthChunk::SerializeKeys() const {
+ const auto data_out = std::make_shared<MemorySegmentOutputStream>(
+ MemorySegmentOutputStream::DEFAULT_SEGMENT_SIZE, pool_);
+ PAIMON_RETURN_NOT_OK(MemorySegmentUtils::CopyToStream(
+ offsets_stream_out_->Segments(), 0,
+ static_cast<int32_t>(offsets_stream_out_->CurrentSize()),
data_out.get()));
+ PAIMON_RETURN_NOT_OK(MemorySegmentUtils::CopyToStream(
+ keys_stream_out_->Segments(), 0,
static_cast<int32_t>(keys_stream_out_->CurrentSize()),
+ data_out.get()));
+ return MemorySegmentUtils::CopyToBytes(
+ data_out->Segments(), 0,
static_cast<int32_t>(data_out->CurrentSize()), pool_.get());
+}
+
+/// Read path
+VariableLengthChunk::VariableLengthChunk(Literal key, int32_t code, int32_t
offset, int32_t size,
+ const std::shared_ptr<KeyFactory>&
factory,
+ const std::shared_ptr<InputStream>&
input_stream,
+ int32_t keys_base_offset, int32_t
offsets_length,
+ int32_t keys_length,
+ const std::shared_ptr<MemoryPool>&
pool)
+ : pool_(pool),
+ key_(std::move(key)),
+ code_(code),
+ offset_(offset),
+ size_(size),
+ factory_(factory),
+ input_stream_(input_stream),
+ keys_base_offset_(keys_base_offset),
+ offsets_length_(offsets_length),
+ keys_length_(keys_length),
+ deserializer_({}),
+ serializer_({}),
+ remaining_offsets_size_(0),
+ remaining_keys_size_(0) {}
+
+/// Write path
+VariableLengthChunk::VariableLengthChunk(Literal key, int32_t code, int32_t
keys_length_limit,
+ const std::shared_ptr<KeyFactory>&
factory,
+ const LiteralSerDeUtils::Serializer&
serializer,
+ const std::shared_ptr<MemoryPool>&
pool)
+ : pool_(pool),
+ key_(std::move(key)),
+ code_(code),
+ offset_(0),
+ size_(0),
+ factory_(factory),
+ keys_base_offset_(0),
+ offsets_length_(0),
+ keys_length_(0),
+ deserializer_({}),
+ serializer_(serializer),
+ offsets_stream_out_(std::make_shared<MemorySegmentOutputStream>(
+ MemorySegmentOutputStream::DEFAULT_SEGMENT_SIZE, pool)),
+ keys_stream_out_(std::make_shared<MemorySegmentOutputStream>(
+ MemorySegmentOutputStream::DEFAULT_SEGMENT_SIZE, pool)),
+ remaining_offsets_size_(keys_length_limit),
+ remaining_keys_size_(keys_length_limit) {}
+
+} // namespace paimon
diff --git
a/src/paimon/common/file_index/rangebitmap/dictionary/variable_length_chunk.h
b/src/paimon/common/file_index/rangebitmap/dictionary/variable_length_chunk.h
new file mode 100644
index 00000000..0c173327
--- /dev/null
+++
b/src/paimon/common/file_index/rangebitmap/dictionary/variable_length_chunk.h
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <memory>
+
+#include "paimon/common/file_index/rangebitmap/dictionary/chunk.h"
+#include
"paimon/common/file_index/rangebitmap/utils/literal_serialization_utils.h"
+#include "paimon/fs/file_system.h"
+
+namespace paimon {
+
+class DataInputStream;
+class InputStream;
+class KeyFactory;
+class MemoryPool;
+class MemorySegmentOutputStream;
+
+class VariableLengthChunk final : public Chunk {
+ public:
+ Result<bool> TryAdd(const Literal& key) override;
+ Result<int32_t> CompareKey(const Literal& lhs, const Literal& rhs)
override;
+ Result<Literal> GetKey(int32_t index) override;
+
+ const Literal& Key() const override {
+ return key_;
+ }
+ int32_t Code() const override {
+ return code_;
+ }
+ int32_t Offset() const override {
+ return offset_;
+ }
+ void SetOffset(int32_t offset) override {
+ offset_ = offset;
+ }
+ int32_t Size() const override {
+ return size_;
+ }
+
+ Result<PAIMON_UNIQUE_PTR<Bytes>> SerializeChunk() const override;
+ Result<PAIMON_UNIQUE_PTR<Bytes>> SerializeKeys() const override;
+
+ // For Read Path
+ VariableLengthChunk(Literal key, int32_t code, int32_t offset, int32_t
size,
+ const std::shared_ptr<KeyFactory>& factory,
+ const std::shared_ptr<InputStream>& input_stream,
int32_t keys_base_offset,
+ int32_t offsets_length, int32_t keys_length,
+ const std::shared_ptr<MemoryPool>& pool);
+
+ // For Write Path
+ VariableLengthChunk(Literal key, int32_t code, int32_t keys_length_limit,
+ const std::shared_ptr<KeyFactory>& factory,
+ const LiteralSerDeUtils::Serializer& serializer,
+ const std::shared_ptr<MemoryPool>& pool);
+
+ public:
+ static constexpr int8_t kCurrentVersion = 1;
+
+ private:
+ Status LoadKeys();
+
+ std::shared_ptr<MemoryPool> pool_;
+ Literal key_; // representative key for binary search
+ int32_t code_; // first code in this chunk
+ int32_t offset_; // offset of this chunk
+ int32_t size_; // number of keys in this chunk
+ std::shared_ptr<KeyFactory> factory_;
+
+ // For read path lazy keys loading
+ std::shared_ptr<InputStream> input_stream_;
+ int32_t keys_base_offset_;
+ int32_t offsets_length_;
+ int32_t keys_length_;
+ PAIMON_UNIQUE_PTR<Bytes> offsets_;
+ PAIMON_UNIQUE_PTR<Bytes> keys_;
+ LiteralSerDeUtils::Deserializer deserializer_;
+
+ // For write path
+ LiteralSerDeUtils::Serializer serializer_;
+ std::shared_ptr<MemorySegmentOutputStream> offsets_stream_out_;
+ std::shared_ptr<MemorySegmentOutputStream> keys_stream_out_;
+ int32_t remaining_offsets_size_;
+ int32_t remaining_keys_size_;
+};
+
+} // namespace paimon
diff --git
a/src/paimon/common/file_index/rangebitmap/range_bitmap_file_index.cpp
b/src/paimon/common/file_index/rangebitmap/range_bitmap_file_index.cpp
index 88e081bd..8384e7df 100644
--- a/src/paimon/common/file_index/rangebitmap/range_bitmap_file_index.cpp
+++ b/src/paimon/common/file_index/rangebitmap/range_bitmap_file_index.cpp
@@ -22,12 +22,12 @@
#include <arrow/type.h>
#include "paimon/common/file_index/rangebitmap/range_bitmap.h"
+#include "paimon/common/file_index/rangebitmap/range_bitmap_type_adapter.h"
#include "paimon/common/io/offset_input_stream.h"
#include "paimon/common/options/memory_size.h"
#include "paimon/common/predicate/literal_converter.h"
#include "paimon/common/utils/arrow/status_utils.h"
#include "paimon/common/utils/checked_cast.h"
-#include "paimon/common/utils/field_type_utils.h"
#include "paimon/file_index/bitmap_index_result.h"
#include "paimon/predicate/literal.h"
#include "paimon/result.h"
@@ -66,10 +66,10 @@ Result<std::shared_ptr<FileIndexWriter>>
RangeBitmapFileIndex::CreateWriter(
Result<std::shared_ptr<RangeBitmapFileIndexWriter>>
RangeBitmapFileIndexWriter::Create(
const std::shared_ptr<arrow::Field>& field, const std::map<std::string,
std::string>& options,
const std::shared_ptr<MemoryPool>& pool) {
- PAIMON_ASSIGN_OR_RAISE(FieldType field_type,
-
FieldTypeUtils::ConvertToFieldType(field->type()->id()));
+ PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<RangeBitmapTypeAdapter>
type_adapter,
+ RangeBitmapTypeAdapter::Create(field->type()));
PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<KeyFactory> shared_key_factory,
- KeyFactory::Create(field_type));
+ KeyFactory::Create(type_adapter->GetStorageType()));
PAIMON_ASSIGN_OR_RAISE(int64_t parsed_chunk_size,
MemorySize::ParseBytes(KeyFactory::kDefaultChunkSize));
if (const auto chunk_size_it =
options.find(RangeBitmapFileIndex::kChunkSize);
@@ -80,10 +80,12 @@ Result<std::shared_ptr<RangeBitmapFileIndexWriter>>
RangeBitmapFileIndexWriter::
PAIMON_ASSIGN_OR_RAISE(
std::unique_ptr<RangeBitmap::Appender> appender_ptr,
RangeBitmap::Appender::Create(shared_key_factory, parsed_chunk_size,
pool));
- return std::make_shared<RangeBitmapFileIndexWriter>(struct_type, pool,
shared_key_factory,
-
std::move(appender_ptr));
+ return std::shared_ptr<RangeBitmapFileIndexWriter>(new
RangeBitmapFileIndexWriter(
+ struct_type, std::move(type_adapter), std::move(appender_ptr)));
}
+RangeBitmapFileIndexWriter::~RangeBitmapFileIndexWriter() = default;
+
Status RangeBitmapFileIndexWriter::AddBatch(::ArrowArray* batch) {
PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Array> array,
arrow::ImportArray(batch, struct_type_));
@@ -91,8 +93,9 @@ Status RangeBitmapFileIndexWriter::AddBatch(::ArrowArray*
batch) {
PAIMON_ASSIGN_OR_RAISE(std::vector<Literal> array_values,
LiteralConverter::ConvertLiteralsFromArray(*(struct_array->field(0)),
/*own_data=*/true));
- for (const auto& literal : array_values) {
- appender_->Append(literal);
+ for (const Literal& literal : array_values) {
+ PAIMON_ASSIGN_OR_RAISE(Literal converted_literal,
type_adapter_->ToStorageLiteral(literal));
+ appender_->Append(converted_literal);
}
return Status::OK();
}
@@ -102,58 +105,70 @@ Result<PAIMON_UNIQUE_PTR<Bytes>>
RangeBitmapFileIndexWriter::SerializedBytes() c
}
RangeBitmapFileIndexWriter::RangeBitmapFileIndexWriter(
- const std::shared_ptr<arrow::DataType>& struct_type, const
std::shared_ptr<MemoryPool>& pool,
- const std::shared_ptr<KeyFactory>& key_factory,
std::unique_ptr<RangeBitmap::Appender> appender)
+ const std::shared_ptr<arrow::DataType>& struct_type,
+ std::unique_ptr<RangeBitmapTypeAdapter> type_adapter,
+ std::unique_ptr<RangeBitmap::Appender> appender)
: struct_type_(struct_type),
- pool_(pool),
- key_factory_(key_factory),
+ type_adapter_(std::move(type_adapter)),
appender_(std::move(appender)) {}
Result<std::shared_ptr<RangeBitmapFileIndexReader>>
RangeBitmapFileIndexReader::Create(
const std::shared_ptr<arrow::DataType>& arrow_type, const int32_t start,
const int32_t length,
const std::shared_ptr<InputStream>& input_stream, const
std::shared_ptr<MemoryPool>& pool) {
- PAIMON_ASSIGN_OR_RAISE(FieldType field_type,
-
FieldTypeUtils::ConvertToFieldType(arrow_type->id()));
+ PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<RangeBitmapTypeAdapter>
type_adapter,
+ RangeBitmapTypeAdapter::Create(arrow_type));
PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<OffsetInputStream> bounded_stream,
OffsetInputStream::Create(input_stream, length,
start));
- PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<RangeBitmap> range_bitmap,
- RangeBitmap::Create(bounded_stream, 0, field_type,
pool));
+ PAIMON_ASSIGN_OR_RAISE(
+ std::unique_ptr<RangeBitmap> range_bitmap,
+ RangeBitmap::Create(bounded_stream, 0, type_adapter->GetStorageType(),
pool));
return std::shared_ptr<RangeBitmapFileIndexReader>(
- new RangeBitmapFileIndexReader(std::move(range_bitmap)));
+ new RangeBitmapFileIndexReader(std::move(type_adapter),
std::move(range_bitmap)));
}
-RangeBitmapFileIndexReader::RangeBitmapFileIndexReader(std::unique_ptr<RangeBitmap>
range_bitmap)
- : range_bitmap_(std::move(range_bitmap)) {}
+RangeBitmapFileIndexReader::~RangeBitmapFileIndexReader() = default;
+
+RangeBitmapFileIndexReader::RangeBitmapFileIndexReader(
+ std::unique_ptr<RangeBitmapTypeAdapter> type_adapter,
std::unique_ptr<RangeBitmap> range_bitmap)
+ : type_adapter_(std::move(type_adapter)),
range_bitmap_(std::move(range_bitmap)) {}
Result<std::shared_ptr<FileIndexResult>>
RangeBitmapFileIndexReader::VisitEqual(
const Literal& literal) {
+ PAIMON_ASSIGN_OR_RAISE(Literal converted_literal,
type_adapter_->ToStorageLiteral(literal));
return std::make_shared<BitmapIndexResult>(
- [self = shared_from_this(), literal]() -> Result<RoaringBitmap32> {
- return self->range_bitmap_->Eq(literal);
+ [self = shared_from_this(), converted_literal]() ->
Result<RoaringBitmap32> {
+ return self->range_bitmap_->Eq(converted_literal);
});
}
Result<std::shared_ptr<FileIndexResult>>
RangeBitmapFileIndexReader::VisitNotEqual(
const Literal& literal) {
+ PAIMON_ASSIGN_OR_RAISE(Literal converted_literal,
type_adapter_->ToStorageLiteral(literal));
return std::make_shared<BitmapIndexResult>(
- [self = shared_from_this(), literal]() -> Result<RoaringBitmap32> {
- return self->range_bitmap_->Neq(literal);
+ [self = shared_from_this(), converted_literal]() ->
Result<RoaringBitmap32> {
+ return self->range_bitmap_->Neq(converted_literal);
});
}
Result<std::shared_ptr<FileIndexResult>> RangeBitmapFileIndexReader::VisitIn(
const std::vector<Literal>& literals) {
+ PAIMON_ASSIGN_OR_RAISE(std::vector<Literal> converted_literals,
+ type_adapter_->ToStorageLiterals(literals));
return std::make_shared<BitmapIndexResult>(
- [self = shared_from_this(), literals]() -> Result<RoaringBitmap32> {
- return self->range_bitmap_->In(literals);
+ [self = shared_from_this(),
+ converted_literals = std::move(converted_literals)]() ->
Result<RoaringBitmap32> {
+ return self->range_bitmap_->In(converted_literals);
});
}
Result<std::shared_ptr<FileIndexResult>>
RangeBitmapFileIndexReader::VisitNotIn(
const std::vector<Literal>& literals) {
+ PAIMON_ASSIGN_OR_RAISE(std::vector<Literal> converted_literals,
+ type_adapter_->ToStorageLiterals(literals));
return std::make_shared<BitmapIndexResult>(
- [self = shared_from_this(), literals]() -> Result<RoaringBitmap32> {
- return self->range_bitmap_->NotIn(literals);
+ [self = shared_from_this(),
+ converted_literals = std::move(converted_literals)]() ->
Result<RoaringBitmap32> {
+ return self->range_bitmap_->NotIn(converted_literals);
});
}
@@ -173,33 +188,37 @@ Result<std::shared_ptr<FileIndexResult>>
RangeBitmapFileIndexReader::VisitIsNotN
Result<std::shared_ptr<FileIndexResult>>
RangeBitmapFileIndexReader::VisitGreaterThan(
const Literal& literal) {
+ PAIMON_ASSIGN_OR_RAISE(Literal converted_literal,
type_adapter_->ToStorageLiteral(literal));
return std::make_shared<BitmapIndexResult>(
- [self = shared_from_this(), literal]() -> Result<RoaringBitmap32> {
- return self->range_bitmap_->Gt(literal);
+ [self = shared_from_this(), converted_literal]() ->
Result<RoaringBitmap32> {
+ return self->range_bitmap_->Gt(converted_literal);
});
}
Result<std::shared_ptr<FileIndexResult>>
RangeBitmapFileIndexReader::VisitLessThan(
const Literal& literal) {
+ PAIMON_ASSIGN_OR_RAISE(Literal converted_literal,
type_adapter_->ToStorageLiteral(literal));
return std::make_shared<BitmapIndexResult>(
- [self = shared_from_this(), literal]() -> Result<RoaringBitmap32> {
- return self->range_bitmap_->Lt(literal);
+ [self = shared_from_this(), converted_literal]() ->
Result<RoaringBitmap32> {
+ return self->range_bitmap_->Lt(converted_literal);
});
}
Result<std::shared_ptr<FileIndexResult>>
RangeBitmapFileIndexReader::VisitGreaterOrEqual(
const Literal& literal) {
+ PAIMON_ASSIGN_OR_RAISE(Literal converted_literal,
type_adapter_->ToStorageLiteral(literal));
return std::make_shared<BitmapIndexResult>(
- [self = shared_from_this(), literal]() -> Result<RoaringBitmap32> {
- return self->range_bitmap_->Gte(literal);
+ [self = shared_from_this(), converted_literal]() ->
Result<RoaringBitmap32> {
+ return self->range_bitmap_->Gte(converted_literal);
});
}
Result<std::shared_ptr<FileIndexResult>>
RangeBitmapFileIndexReader::VisitLessOrEqual(
const Literal& literal) {
+ PAIMON_ASSIGN_OR_RAISE(Literal converted_literal,
type_adapter_->ToStorageLiteral(literal));
return std::make_shared<BitmapIndexResult>(
- [self = shared_from_this(), literal]() -> Result<RoaringBitmap32> {
- return self->range_bitmap_->Lte(literal);
+ [self = shared_from_this(), converted_literal]() ->
Result<RoaringBitmap32> {
+ return self->range_bitmap_->Lte(converted_literal);
});
}
diff --git a/src/paimon/common/file_index/rangebitmap/range_bitmap_file_index.h
b/src/paimon/common/file_index/rangebitmap/range_bitmap_file_index.h
index 64289a53..3e7d511e 100644
--- a/src/paimon/common/file_index/rangebitmap/range_bitmap_file_index.h
+++ b/src/paimon/common/file_index/rangebitmap/range_bitmap_file_index.h
@@ -36,6 +36,7 @@ namespace paimon {
class RangeBitmapFileIndexWriter;
class RangeBitmapFileIndexReader;
+class RangeBitmapTypeAdapter;
class PAIMON_EXPORT RangeBitmapFileIndex final : public FileIndexer {
public:
@@ -64,20 +65,19 @@ class RangeBitmapFileIndexWriter final : public
FileIndexWriter {
const std::shared_ptr<arrow::Field>& field,
const std::map<std::string, std::string>& options, const
std::shared_ptr<MemoryPool>& pool);
+ ~RangeBitmapFileIndexWriter() override;
+
Status AddBatch(::ArrowArray* batch) override;
Result<PAIMON_UNIQUE_PTR<Bytes>> SerializedBytes() const override;
+ private:
RangeBitmapFileIndexWriter(const std::shared_ptr<arrow::DataType>&
struct_type,
- const std::shared_ptr<MemoryPool>& pool,
- const std::shared_ptr<KeyFactory>& key_factory,
+ std::unique_ptr<RangeBitmapTypeAdapter>
type_adapter,
std::unique_ptr<RangeBitmap::Appender>
appender);
- private:
- /// @note struct_type_ contains only one field with arrow_type_, used for
import from C
- /// interface.
+ /// @note struct_type_ contains only the indexed field and is used to
import Arrow C data.
std::shared_ptr<arrow::DataType> struct_type_;
- std::shared_ptr<MemoryPool> pool_;
- std::shared_ptr<KeyFactory> key_factory_;
+ std::unique_ptr<RangeBitmapTypeAdapter> type_adapter_;
std::unique_ptr<RangeBitmap::Appender> appender_;
};
@@ -89,8 +89,11 @@ class RangeBitmapFileIndexReader final
const std::shared_ptr<arrow::DataType>& arrow_type, int32_t start,
int32_t length,
const std::shared_ptr<InputStream>& input_stream, const
std::shared_ptr<MemoryPool>& pool);
+ ~RangeBitmapFileIndexReader() override;
+
private:
- explicit RangeBitmapFileIndexReader(std::unique_ptr<RangeBitmap>
range_bitmap);
+ RangeBitmapFileIndexReader(std::unique_ptr<RangeBitmapTypeAdapter>
type_adapter,
+ std::unique_ptr<RangeBitmap> range_bitmap);
Result<std::shared_ptr<FileIndexResult>> VisitEqual(const Literal&
literal) override;
Result<std::shared_ptr<FileIndexResult>> VisitNotEqual(const Literal&
literal) override;
@@ -104,6 +107,7 @@ class RangeBitmapFileIndexReader final
Result<std::shared_ptr<FileIndexResult>> VisitGreaterOrEqual(const
Literal& literal) override;
Result<std::shared_ptr<FileIndexResult>> VisitLessOrEqual(const Literal&
literal) override;
+ std::unique_ptr<RangeBitmapTypeAdapter> type_adapter_;
std::unique_ptr<RangeBitmap> range_bitmap_;
};
diff --git
a/src/paimon/common/file_index/rangebitmap/range_bitmap_file_index_test.cpp
b/src/paimon/common/file_index/rangebitmap/range_bitmap_file_index_test.cpp
index 6157ee02..794f25c5 100644
--- a/src/paimon/common/file_index/rangebitmap/range_bitmap_file_index_test.cpp
+++ b/src/paimon/common/file_index/rangebitmap/range_bitmap_file_index_test.cpp
@@ -20,13 +20,18 @@
#include <gtest/gtest.h>
+#include <limits>
#include <memory>
#include <numeric>
#include <set>
#include "arrow/api.h"
#include "arrow/c/bridge.h"
+#include "arrow/ipc/json_simple.h"
#include "paimon/common/utils/arrow/status_utils.h"
+#include "paimon/common/utils/math.h"
+#include "paimon/data/decimal.h"
+#include "paimon/data/timestamp.h"
#include "paimon/file_index/bitmap_index_result.h"
#include "paimon/file_index/file_index_format.h"
#include "paimon/file_index/file_indexer_factory.h"
@@ -98,6 +103,17 @@ class RangeBitmapFileIndexTest : public ::testing::Test {
const std::set<int32_t>& null_indices, const std::map<std::string,
std::string>& options,
PAIMON_UNIQUE_PTR<Bytes>* serialized_bytes_out);
+ Result<std::shared_ptr<RangeBitmapFileIndexReader>> CreateReaderFromJson(
+ const std::shared_ptr<arrow::DataType>& arrow_type, const std::string&
json,
+ const std::map<std::string, std::string>& options,
+ PAIMON_UNIQUE_PTR<Bytes>* serialized_bytes_out);
+
+ Result<std::shared_ptr<RangeBitmapFileIndexReader>> CreateReaderFromArray(
+ const std::shared_ptr<arrow::DataType>& arrow_type,
+ const std::shared_ptr<arrow::Array>& array,
+ const std::map<std::string, std::string>& options,
+ PAIMON_UNIQUE_PTR<Bytes>* serialized_bytes_out);
+
protected:
std::shared_ptr<MemoryPool> pool_;
@@ -122,11 +138,18 @@ Result<std::shared_ptr<RangeBitmapFileIndexReader>>
RangeBitmapFileIndexTest::Cr
}
std::shared_ptr<arrow::Array> arrow_array;
PAIMON_RETURN_NOT_OK_FROM_ARROW(builder->Finish(&arrow_array));
+ return CreateReaderFromArray(arrow_type, arrow_array, options,
serialized_bytes_out);
+}
+
+Result<std::shared_ptr<RangeBitmapFileIndexReader>>
RangeBitmapFileIndexTest::CreateReaderFromArray(
+ const std::shared_ptr<arrow::DataType>& arrow_type, const
std::shared_ptr<arrow::Array>& array,
+ const std::map<std::string, std::string>& options,
+ PAIMON_UNIQUE_PTR<Bytes>* serialized_bytes_out) {
// Wrap in StructArray (single field) as required by
RangeBitmapFileIndexWriter
auto field = arrow::field("test_field", arrow_type);
arrow::FieldVector fields = {field};
PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::StructArray>
struct_array,
- arrow::StructArray::Make({arrow_array},
fields));
+ arrow::StructArray::Make({array},
fields));
auto c_array = std::make_unique<::ArrowArray>();
PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*struct_array,
c_array.get()));
// Create writer
@@ -149,6 +172,18 @@ Result<std::shared_ptr<RangeBitmapFileIndexReader>>
RangeBitmapFileIndexTest::Cr
return reader;
}
+Result<std::shared_ptr<RangeBitmapFileIndexReader>>
RangeBitmapFileIndexTest::CreateReaderFromJson(
+ const std::shared_ptr<arrow::DataType>& arrow_type, const std::string&
json,
+ const std::map<std::string, std::string>& options,
+ PAIMON_UNIQUE_PTR<Bytes>* serialized_bytes_out) {
+ const auto field = arrow::field("test_field", arrow_type);
+ PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+ std::shared_ptr<arrow::Array> array,
+ arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({field}),
json));
+ const auto struct_array = checked_pointer_cast<arrow::StructArray>(array);
+ return CreateReaderFromArray(arrow_type, struct_array->field(0), options,
serialized_bytes_out);
+}
+
// Test with all NULL values
TEST_F(RangeBitmapFileIndexTest, TestAllNullValues) {
constexpr int32_t num_rows = 10;
@@ -532,6 +567,106 @@ TEST_F(RangeBitmapFileIndexTest,
TestWriteAndReadRangeBitmapIndexDouble) {
CheckResult(is_not_null_result, all_positions);
}
+TEST_F(RangeBitmapFileIndexTest, TestFloatingPointSpecialValues) {
+ const std::vector<int32_t> nan_positions = {0, 1, 2, 5};
+ const std::vector<int32_t> negative_zero_positions = {3, 6};
+ const std::vector<int32_t> positive_zero_positions = {4, 7};
+ const std::vector<int32_t> non_nan_positions = {3, 4, 6, 7, 8, 9, 10, 11};
+ const std::vector<int32_t> less_than_positive_zero_positions = {3, 6, 8,
10};
+ const std::vector<int32_t> nan_and_negative_zero_positions = {0, 1, 2, 3,
5, 6};
+ const std::vector<int32_t> non_nan_and_non_negative_zero_positions = {4,
7, 8, 9, 10, 11};
+
+ const auto check_reader = [&](const
std::shared_ptr<RangeBitmapFileIndexReader>& reader,
+ const std::vector<Literal>& nan_literals,
+ const Literal& negative_zero, const Literal&
positive_zero,
+ const Literal& positive_infinity) {
+ for (const Literal& nan_literal : nan_literals) {
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<FileIndexResult> result,
+ reader->VisitEqual(nan_literal));
+ CheckResult(result, nan_positions);
+ }
+
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<FileIndexResult>
negative_zero_result,
+ reader->VisitEqual(negative_zero));
+ CheckResult(negative_zero_result, negative_zero_positions);
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<FileIndexResult>
positive_zero_result,
+ reader->VisitEqual(positive_zero));
+ CheckResult(positive_zero_result, positive_zero_positions);
+
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<FileIndexResult>
less_than_nan_result,
+ reader->VisitLessThan(nan_literals.front()));
+ CheckResult(less_than_nan_result, non_nan_positions);
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<FileIndexResult>
greater_than_infinity_result,
+ reader->VisitGreaterThan(positive_infinity));
+ CheckResult(greater_than_infinity_result, nan_positions);
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<FileIndexResult>
less_than_positive_zero_result,
+ reader->VisitLessThan(positive_zero));
+ CheckResult(less_than_positive_zero_result,
less_than_positive_zero_positions);
+
+ const std::vector<Literal> nan_and_negative_zero = {nan_literals[1],
negative_zero};
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<FileIndexResult> in_result,
+ reader->VisitIn(nan_and_negative_zero));
+ CheckResult(in_result, nan_and_negative_zero_positions);
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<FileIndexResult> not_in_result,
+ reader->VisitNotIn(nan_and_negative_zero));
+ CheckResult(not_in_result, non_nan_and_non_negative_zero_positions);
+ };
+
+ const auto float_nan =
FloatingPointFromBits<float>(kCanonicalFloatNaNBits);
+ const auto float_positive_payload_nan =
FloatingPointFromBits<float>(uint32_t{0x7fc12345});
+ const auto float_negative_payload_nan =
FloatingPointFromBits<float>(uint32_t{0xffc54321});
+ const std::vector<float> float_values = {
+ float_nan,
+ float_positive_payload_nan,
+ float_negative_payload_nan,
+ -0.0f,
+ +0.0f,
+ float_negative_payload_nan,
+ -0.0f,
+ +0.0f,
+ -std::numeric_limits<float>::infinity(),
+ std::numeric_limits<float>::infinity(),
+ -1.0f,
+ +1.0f,
+ };
+ PAIMON_UNIQUE_PTR<Bytes> float_serialized_bytes;
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<RangeBitmapFileIndexReader>
float_reader,
+ (CreateReaderForTest<arrow::FloatBuilder, float>(
+ arrow::float32(), float_values,
&float_serialized_bytes)));
+ check_reader(float_reader,
+ {Literal(float_nan), Literal(float_positive_payload_nan),
+ Literal(float_negative_payload_nan)},
+ Literal(-0.0f), Literal(+0.0f),
Literal(std::numeric_limits<float>::infinity()));
+
+ const auto double_nan =
FloatingPointFromBits<double>(kCanonicalDoubleNaNBits);
+ const auto double_positive_payload_nan =
+ FloatingPointFromBits<double>(uint64_t{0x7ff8123456789abc});
+ const auto double_negative_payload_nan =
+ FloatingPointFromBits<double>(uint64_t{0xfff8abcdef012345});
+ const std::vector<double> double_values = {
+ double_nan,
+ double_positive_payload_nan,
+ double_negative_payload_nan,
+ -0.0,
+ +0.0,
+ double_negative_payload_nan,
+ -0.0,
+ +0.0,
+ -std::numeric_limits<double>::infinity(),
+ std::numeric_limits<double>::infinity(),
+ -1.0,
+ +1.0,
+ };
+ PAIMON_UNIQUE_PTR<Bytes> double_serialized_bytes;
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<RangeBitmapFileIndexReader>
double_reader,
+ (CreateReaderForTest<arrow::DoubleBuilder, double>(
+ arrow::float64(), double_values,
&double_serialized_bytes)));
+ check_reader(double_reader,
+ {Literal(double_nan), Literal(double_positive_payload_nan),
+ Literal(double_negative_payload_nan)},
+ Literal(-0.0), Literal(+0.0),
Literal(std::numeric_limits<double>::infinity()));
+}
+
TEST_F(RangeBitmapFileIndexTest, TestWriteAndReadRangeBitmapIndexDate) {
std::vector<int32_t> test_data = {42432, 24649, 42432, 38001, 24649,
50000, 12000};
const auto& arrow_type = arrow::date32();
@@ -580,6 +715,56 @@ TEST_F(RangeBitmapFileIndexTest,
TestWriteAndReadRangeBitmapIndexDate) {
CheckResult(is_not_null_result, all_positions);
}
+TEST_F(RangeBitmapFileIndexTest, TestWriteAndReadStringDecimalAndTimestamp) {
+ {
+ const auto type = arrow::utf8();
+ PAIMON_UNIQUE_PTR<Bytes> serialized_bytes;
+ ASSERT_OK_AND_ASSIGN(
+ auto reader,
+ CreateReaderFromJson(type, R"([["pear"], ["apple"], [null],
["banana"], ["apple"]])",
+ {{"chunk-size", "12b"}}, &serialized_bytes));
+ const Literal apple(FieldType::STRING, "apple", 5);
+ const Literal banana(FieldType::STRING, "banana", 6);
+ CheckResult(reader->VisitEqual(apple).value(), {1, 4});
+ CheckResult(reader->VisitGreaterOrEqual(banana).value(), {0, 3});
+ CheckResult(reader->VisitIsNull().value(), {2});
+ }
+ {
+ const auto type = arrow::decimal128(10, 2);
+ PAIMON_UNIQUE_PTR<Bytes> serialized_bytes;
+ ASSERT_OK_AND_ASSIGN(
+ auto reader,
+ CreateReaderFromJson(type, R"([["1.00"], ["2.50"], [null],
["-1.25"], ["2.50"]])", {},
+ &serialized_bytes));
+ CheckResult(reader->VisitEqual(Literal(Decimal(10, 2, 250))).value(),
{1, 4});
+ CheckResult(reader->VisitLessThan(Literal(Decimal(10, 2, 0))).value(),
{3});
+ CheckResult(reader->VisitIsNull().value(), {2});
+
+ // Range Bitmap does not rescale Decimal literals. A mathematically
equivalent literal
+ // with a different scale produces an incorrect empty result, so
callers must use the
+ // field's scale.
+ CheckResult(reader->VisitEqual(Literal(Decimal(10, 3, 2500))).value(),
{});
+ }
+ {
+ const auto type = arrow::timestamp(arrow::TimeUnit::MICRO);
+ PAIMON_UNIQUE_PTR<Bytes> serialized_bytes;
+ ASSERT_OK_AND_ASSIGN(
+ auto reader,
+ CreateReaderFromJson(type, R"([[1000001], [2000002], [null],
[-1000001], [2000002]])",
+ {}, &serialized_bytes));
+ CheckResult(reader->VisitEqual(Literal(Timestamp(2000,
2000))).value(), {1, 4});
+ CheckResult(reader->VisitLessThan(Literal(Timestamp(0, 0))).value(),
{3});
+ CheckResult(reader->VisitIsNull().value(), {2});
+ }
+
+ ASSERT_NOK_WITH_MSG(
+ RangeBitmapFileIndexWriter::Create(arrow::field("f0",
arrow::decimal128(19, 2)), {}, pool_),
+ "DECIMAL with precision in [1, 18]");
+ ASSERT_NOK_WITH_MSG(RangeBitmapFileIndexWriter::Create(
+ arrow::field("f0",
arrow::timestamp(arrow::TimeUnit::NANO)), {}, pool_),
+ "TIMESTAMP with precision in [0, 6]");
+}
+
TEST_F(RangeBitmapFileIndexTest, TestRangeBitmapEdgeCases) {
// Scope 1: All values identical
{
diff --git
a/src/paimon/common/file_index/rangebitmap/range_bitmap_type_adapter.cpp
b/src/paimon/common/file_index/rangebitmap/range_bitmap_type_adapter.cpp
new file mode 100644
index 00000000..e9157ec1
--- /dev/null
+++ b/src/paimon/common/file_index/rangebitmap/range_bitmap_type_adapter.cpp
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "paimon/common/file_index/rangebitmap/range_bitmap_type_adapter.h"
+
+#include "arrow/type.h"
+#include "fmt/format.h"
+#include "paimon/common/utils/checked_cast.h"
+#include "paimon/common/utils/date_time_utils.h"
+#include "paimon/common/utils/field_type_utils.h"
+#include "paimon/data/decimal.h"
+#include "paimon/data/timestamp.h"
+#include "paimon/status.h"
+
+namespace paimon {
+
+Result<std::unique_ptr<RangeBitmapTypeAdapter>> RangeBitmapTypeAdapter::Create(
+ const std::shared_ptr<arrow::DataType>& arrow_type) {
+ PAIMON_ASSIGN_OR_RAISE(FieldType field_type,
+
FieldTypeUtils::ConvertToFieldType(arrow_type->id()));
+ if (field_type == FieldType::DECIMAL) {
+ const auto decimal_type =
checked_pointer_cast<arrow::Decimal128Type>(arrow_type);
+ if (decimal_type->precision() > 18) {
+ return Status::Invalid(fmt::format(
+ "range-bitmap index only supports DECIMAL with precision in
[1, 18], got {}",
+ decimal_type->precision()));
+ }
+ return std::unique_ptr<RangeBitmapTypeAdapter>(
+ new RangeBitmapTypeAdapter(field_type, FieldType::BIGINT,
std::nullopt));
+ }
+ if (field_type == FieldType::TIMESTAMP) {
+ const auto timestamp_type =
checked_pointer_cast<arrow::TimestampType>(arrow_type);
+ const int32_t precision =
DateTimeUtils::GetPrecisionFromType(timestamp_type);
+ if (precision > 6) {
+ return Status::Invalid(fmt::format(
+ "range-bitmap index only supports TIMESTAMP with precision in
[0, 6], got {}",
+ precision));
+ }
+ return std::unique_ptr<RangeBitmapTypeAdapter>(
+ new RangeBitmapTypeAdapter(field_type, FieldType::BIGINT,
precision));
+ }
+ return std::unique_ptr<RangeBitmapTypeAdapter>(
+ new RangeBitmapTypeAdapter(field_type, field_type, std::nullopt));
+}
+
+FieldType RangeBitmapTypeAdapter::GetStorageType() const {
+ return storage_type_;
+}
+
+Result<Literal> RangeBitmapTypeAdapter::ToStorageLiteral(const Literal&
literal) const {
+ if (literal.IsNull()) {
+ return Literal(storage_type_);
+ }
+ if (logical_type_ == FieldType::DECIMAL) {
+ if (literal.GetType() != FieldType::DECIMAL) {
+ return Status::Invalid("range-bitmap DECIMAL field requires a
DECIMAL literal");
+ }
+ return Literal(literal.GetValue<Decimal>().ToUnscaledLong());
+ }
+ if (logical_type_ == FieldType::TIMESTAMP) {
+ if (literal.GetType() != FieldType::TIMESTAMP) {
+ return Status::Invalid("range-bitmap TIMESTAMP field requires a
TIMESTAMP literal");
+ }
+ if (!timestamp_precision_.has_value()) {
+ return Status::Invalid("range-bitmap TIMESTAMP adapter is missing
precision");
+ }
+ const auto value = literal.GetValue<Timestamp>();
+ return Literal(*timestamp_precision_ <= Timestamp::MILLIS_PRECISION
+ ? value.GetMillisecond()
+ : value.ToMicrosecond());
+ }
+ if (literal.GetType() != storage_type_) {
+ return Status::Invalid(
+ fmt::format("range-bitmap literal type {} does not match field
type {}",
+ FieldTypeUtils::FieldTypeToString(literal.GetType()),
+ FieldTypeUtils::FieldTypeToString(storage_type_)));
+ }
+ return literal;
+}
+
+Result<std::vector<Literal>> RangeBitmapTypeAdapter::ToStorageLiterals(
+ const std::vector<Literal>& literals) const {
+ std::vector<Literal> converted_literals;
+ converted_literals.reserve(literals.size());
+ for (const Literal& literal : literals) {
+ PAIMON_ASSIGN_OR_RAISE(Literal converted_literal,
ToStorageLiteral(literal));
+ converted_literals.emplace_back(std::move(converted_literal));
+ }
+ return converted_literals;
+}
+
+RangeBitmapTypeAdapter::RangeBitmapTypeAdapter(FieldType logical_type,
FieldType storage_type,
+ std::optional<int32_t>
timestamp_precision)
+ : logical_type_(logical_type),
+ storage_type_(storage_type),
+ timestamp_precision_(timestamp_precision) {}
+
+} // namespace paimon
diff --git
a/src/paimon/common/file_index/rangebitmap/range_bitmap_type_adapter.h
b/src/paimon/common/file_index/rangebitmap/range_bitmap_type_adapter.h
new file mode 100644
index 00000000..bd9cd5bb
--- /dev/null
+++ b/src/paimon/common/file_index/rangebitmap/range_bitmap_type_adapter.h
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <memory>
+#include <optional>
+#include <vector>
+
+#include "paimon/defs.h"
+#include "paimon/predicate/literal.h"
+#include "paimon/result.h"
+
+namespace arrow {
+class DataType;
+} // namespace arrow
+
+namespace paimon {
+
+/// Adapts logical field values to the physical key type stored by
range-bitmap indexes.
+class RangeBitmapTypeAdapter {
+ public:
+ static Result<std::unique_ptr<RangeBitmapTypeAdapter>> Create(
+ const std::shared_ptr<arrow::DataType>& arrow_type);
+
+ FieldType GetStorageType() const;
+
+ Result<Literal> ToStorageLiteral(const Literal& literal) const;
+
+ Result<std::vector<Literal>> ToStorageLiterals(const std::vector<Literal>&
literals) const;
+
+ private:
+ RangeBitmapTypeAdapter(FieldType logical_type, FieldType storage_type,
+ std::optional<int32_t> timestamp_precision);
+
+ FieldType logical_type_;
+ FieldType storage_type_;
+ std::optional<int32_t> timestamp_precision_;
+};
+
+} // namespace paimon
diff --git
a/src/paimon/common/file_index/rangebitmap/range_bitmap_type_adapter_test.cpp
b/src/paimon/common/file_index/rangebitmap/range_bitmap_type_adapter_test.cpp
new file mode 100644
index 00000000..90f7dfbb
--- /dev/null
+++
b/src/paimon/common/file_index/rangebitmap/range_bitmap_type_adapter_test.cpp
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "paimon/common/file_index/rangebitmap/range_bitmap_type_adapter.h"
+
+#include <gtest/gtest.h>
+
+#include "arrow/api.h"
+#include "paimon/data/decimal.h"
+#include "paimon/data/timestamp.h"
+#include "paimon/testing/utils/testharness.h"
+
+namespace paimon::test {
+
+TEST(RangeBitmapTypeAdapterTest, TestStorageType) {
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<RangeBitmapTypeAdapter> int_adapter,
+ RangeBitmapTypeAdapter::Create(arrow::int32()));
+ ASSERT_EQ(FieldType::INT, int_adapter->GetStorageType());
+
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<RangeBitmapTypeAdapter>
string_adapter,
+ RangeBitmapTypeAdapter::Create(arrow::utf8()));
+ ASSERT_EQ(FieldType::STRING, string_adapter->GetStorageType());
+
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<RangeBitmapTypeAdapter>
decimal_adapter,
+ RangeBitmapTypeAdapter::Create(arrow::decimal128(18,
2)));
+ ASSERT_EQ(FieldType::BIGINT, decimal_adapter->GetStorageType());
+
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<RangeBitmapTypeAdapter>
timestamp_adapter,
+
RangeBitmapTypeAdapter::Create(arrow::timestamp(arrow::TimeUnit::MICRO)));
+ ASSERT_EQ(FieldType::BIGINT, timestamp_adapter->GetStorageType());
+
+ ASSERT_NOK_WITH_MSG(RangeBitmapTypeAdapter::Create(arrow::decimal128(19,
2)),
+ "DECIMAL with precision in [1, 18]");
+
ASSERT_NOK_WITH_MSG(RangeBitmapTypeAdapter::Create(arrow::timestamp(arrow::TimeUnit::NANO)),
+ "TIMESTAMP with precision in [0, 6]");
+}
+
+TEST(RangeBitmapTypeAdapterTest, TestDecimalLiteralConversion) {
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<RangeBitmapTypeAdapter> adapter,
+ RangeBitmapTypeAdapter::Create(arrow::decimal128(10,
2)));
+
+ ASSERT_OK_AND_ASSIGN(Literal converted,
+ adapter->ToStorageLiteral(Literal(Decimal(10, 2,
12345))));
+ ASSERT_EQ(FieldType::BIGINT, converted.GetType());
+ ASSERT_EQ(12345, converted.GetValue<int64_t>());
+
+ ASSERT_OK_AND_ASSIGN(Literal converted_null,
+
adapter->ToStorageLiteral(Literal(FieldType::DECIMAL)));
+ ASSERT_EQ(FieldType::BIGINT, converted_null.GetType());
+ ASSERT_TRUE(converted_null.IsNull());
+
+ ASSERT_NOK_WITH_MSG(adapter->ToStorageLiteral(Literal(int64_t{12345})),
+ "DECIMAL field requires a DECIMAL literal");
+}
+
+TEST(RangeBitmapTypeAdapterTest, TestTimestampLiteralConversion) {
+ const Timestamp timestamp(1234, 567000);
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<RangeBitmapTypeAdapter>
millis_adapter,
+
RangeBitmapTypeAdapter::Create(arrow::timestamp(arrow::TimeUnit::MILLI)));
+ ASSERT_OK_AND_ASSIGN(Literal millis,
millis_adapter->ToStorageLiteral(Literal(timestamp)));
+ ASSERT_EQ(FieldType::BIGINT, millis.GetType());
+ ASSERT_EQ(1234, millis.GetValue<int64_t>());
+
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<RangeBitmapTypeAdapter>
micros_adapter,
+
RangeBitmapTypeAdapter::Create(arrow::timestamp(arrow::TimeUnit::MICRO)));
+ ASSERT_OK_AND_ASSIGN(Literal micros,
micros_adapter->ToStorageLiteral(Literal(timestamp)));
+ ASSERT_EQ(FieldType::BIGINT, micros.GetType());
+ ASSERT_EQ(1234567, micros.GetValue<int64_t>());
+
+ ASSERT_OK_AND_ASSIGN(Literal converted_null,
+
micros_adapter->ToStorageLiteral(Literal(FieldType::TIMESTAMP)));
+ ASSERT_EQ(FieldType::BIGINT, converted_null.GetType());
+ ASSERT_TRUE(converted_null.IsNull());
+
+
ASSERT_NOK_WITH_MSG(micros_adapter->ToStorageLiteral(Literal(int64_t{1234567})),
+ "TIMESTAMP field requires a TIMESTAMP literal");
+}
+
+TEST(RangeBitmapTypeAdapterTest, TestLiteralBatchConversion) {
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<RangeBitmapTypeAdapter> adapter,
+ RangeBitmapTypeAdapter::Create(arrow::int32()));
+ const std::vector<Literal> literals = {Literal(int32_t{1}),
Literal(FieldType::INT),
+ Literal(int32_t{3})};
+ ASSERT_OK_AND_ASSIGN(std::vector<Literal> converted,
adapter->ToStorageLiterals(literals));
+ ASSERT_EQ(3, converted.size());
+ ASSERT_EQ(1, converted[0].GetValue<int32_t>());
+ ASSERT_TRUE(converted[1].IsNull());
+ ASSERT_EQ(3, converted[2].GetValue<int32_t>());
+
+ ASSERT_NOK_WITH_MSG(adapter->ToStorageLiterals({Literal(int32_t{1}),
Literal(int64_t{2})}),
+ "literal type BIGINT does not match field type INT");
+}
+
+} // namespace paimon::test
diff --git
a/src/paimon/common/global_index/btree/btree_global_index_integration_test.cpp
b/src/paimon/common/global_index/btree/btree_global_index_integration_test.cpp
index 653b617d..ce25b9b4 100644
---
a/src/paimon/common/global_index/btree/btree_global_index_integration_test.cpp
+++
b/src/paimon/common/global_index/btree/btree_global_index_integration_test.cpp
@@ -1330,6 +1330,14 @@ TEST_P(BTreeGlobalIndexIntegrationTest,
WriteAndReadDecimalCompactData) {
ASSERT_OK_AND_ASSIGN(auto result, reader->VisitEqual(lit_250));
CheckResult(result, {2, 3});
}
+ {
+ // BTree does not rescale Decimal literals. A mathematically
equivalent literal with a
+ // different scale produces an incorrect empty result, so callers must
use the field's
+ // scale.
+ Literal lit_2500(Decimal::FromUnscaledLong(2500, 10, 3));
+ ASSERT_OK_AND_ASSIGN(auto result, reader->VisitEqual(lit_2500));
+ CheckResult(result, {});
+ }
{
Literal lit_250(Decimal::FromUnscaledLong(250, 10, 2));
ASSERT_OK_AND_ASSIGN(auto result, reader->VisitNotEqual(lit_250));
diff --git a/src/paimon/common/predicate/predicate_validator.h
b/src/paimon/common/predicate/predicate_validator.h
index 3242d4bc..7f199f50 100644
--- a/src/paimon/common/predicate/predicate_validator.h
+++ b/src/paimon/common/predicate/predicate_validator.h
@@ -24,8 +24,11 @@
#include <vector>
#include "arrow/type.h"
+#include "arrow/util/decimal.h"
#include "fmt/format.h"
+#include "paimon/common/utils/checked_cast.h"
#include "paimon/common/utils/field_type_utils.h"
+#include "paimon/data/decimal.h"
#include "paimon/defs.h"
#include "paimon/predicate/compound_predicate.h"
#include "paimon/predicate/leaf_predicate.h"
@@ -39,6 +42,14 @@ class PredicateValidator {
PredicateValidator() = delete;
~PredicateValidator() = delete;
+ static Status ValidatePredicateWithSchema(const arrow::Schema& schema,
+ const
std::shared_ptr<Predicate>& predicate,
+ bool validate_field_idx) {
+ PAIMON_RETURN_NOT_OK(ValidatePredicateWithLiterals(predicate));
+ return ValidatePredicateWithSchemaImpl(schema, predicate,
validate_field_idx);
+ }
+
+ private:
static Status ValidatePredicateWithLiterals(const
std::shared_ptr<Predicate>& predicate) {
if (auto leaf_predicate =
std::dynamic_pointer_cast<LeafPredicate>(predicate)) {
const auto& field_name = leaf_predicate->FieldName();
@@ -68,9 +79,9 @@ class PredicateValidator {
return Status::OK();
}
- static Status ValidatePredicateWithSchema(const arrow::Schema& schema,
- const
std::shared_ptr<Predicate>& predicate,
- bool validate_field_idx) {
+ static Status ValidatePredicateWithSchemaImpl(const arrow::Schema& schema,
+ const
std::shared_ptr<Predicate>& predicate,
+ bool validate_field_idx) {
if (auto leaf_predicate =
std::dynamic_pointer_cast<LeafPredicate>(predicate)) {
const auto& field_name = leaf_predicate->FieldName();
// check field index
@@ -86,20 +97,47 @@ class PredicateValidator {
field_name, schema_field_idx,
leaf_predicate->FieldIndex()));
}
// check field type (schema vs. predicate)
+ const std::shared_ptr<arrow::DataType>& schema_type =
+ schema.field(schema_field_idx)->type();
PAIMON_RETURN_NOT_OK(ValidateDataTypeWithSchemaAndPredicate(
- *schema.field(schema_field_idx)->type(),
leaf_predicate->GetFieldType()));
+ *schema_type, leaf_predicate->GetFieldType()));
+ if (schema_type->id() == arrow::Type::DECIMAL128) {
+ PAIMON_RETURN_NOT_OK(ValidateDecimalLiterals(
+ *checked_pointer_cast<arrow::Decimal128Type>(schema_type),
*leaf_predicate));
+ }
} else if (auto compound_predicate =
std::dynamic_pointer_cast<CompoundPredicate>(predicate)) {
const auto& children = compound_predicate->Children();
for (const auto& child : children) {
PAIMON_RETURN_NOT_OK(
- ValidatePredicateWithSchema(schema, child,
validate_field_idx));
+ ValidatePredicateWithSchemaImpl(schema, child,
validate_field_idx));
+ }
+ }
+ return Status::OK();
+ }
+
+ static Status ValidateDecimalLiterals(const arrow::Decimal128Type&
field_type,
+ const LeafPredicate& predicate) {
+ const std::string& field_name = predicate.FieldName();
+ for (const Literal& literal : predicate.Literals()) {
+ const auto decimal = literal.GetValue<Decimal>();
+ if (decimal.Scale() != field_type.scale()) {
+ return Status::Invalid(fmt::format(
+ "decimal literal for field {} has scale {}, expected {};
rescale the literal "
+ "before building the predicate",
+ field_name, decimal.Scale(), field_type.scale()));
+ }
+
+ arrow::Decimal128 unscaled_value(decimal.HighBits(),
decimal.LowBits());
+ if (!unscaled_value.FitsInPrecision(field_type.precision())) {
+ return Status::Invalid(fmt::format(
+ "decimal literal {} for field {} does not fit field type
DECIMAL({}, {})",
+ decimal.ToString(), field_name, field_type.precision(),
field_type.scale()));
}
}
return Status::OK();
}
- private:
static Status ValidateDataTypeWithSchemaAndPredicate(const
arrow::DataType& schema_type,
const FieldType&
field_type) {
const auto kind = schema_type.id();
diff --git a/src/paimon/common/predicate/predicate_validator_test.cpp
b/src/paimon/common/predicate/predicate_validator_test.cpp
index d8342075..75ae141c 100644
--- a/src/paimon/common/predicate/predicate_validator_test.cpp
+++ b/src/paimon/common/predicate/predicate_validator_test.cpp
@@ -33,6 +33,18 @@ class Schema;
namespace paimon::test {
TEST(PredicateValidatorTest, TestValidateLiterals) {
std::string str("apple");
+ std::shared_ptr<arrow::Schema> schema = arrow::schema(arrow::FieldVector({
+ arrow::field("f0", arrow::int64()),
+ arrow::field("f1", arrow::float32()),
+ arrow::field("f2", arrow::utf8()),
+ arrow::field("f3", arrow::boolean()),
+ arrow::field("f4", arrow::float64()),
+ arrow::field("f5", arrow::int8()),
+ arrow::field("f6", arrow::date32()),
+ arrow::field("f7", arrow::timestamp(arrow::TimeUnit::NANO)),
+ arrow::field("f8", arrow::decimal128(23, 5)),
+ arrow::field("f9", arrow::binary()),
+ }));
{
ASSERT_OK_AND_ASSIGN(
auto predicate,
@@ -59,7 +71,8 @@ TEST(PredicateValidatorTest, TestValidateLiterals) {
PredicateBuilder::Equal(/*field_index=*/9,
/*field_name=*/"f9", FieldType::BINARY,
Literal(FieldType::BINARY, str.data(),
str.size())),
}));
-
ASSERT_OK(PredicateValidator::ValidatePredicateWithLiterals(predicate));
+ ASSERT_OK(PredicateValidator::ValidatePredicateWithSchema(*schema,
predicate,
+
/*validate_field_idx=*/true));
}
{
// f1 field type is FLOAT, literal type is BIGINT
@@ -75,9 +88,10 @@ TEST(PredicateValidatorTest, TestValidateLiterals) {
PredicateBuilder::Equal(/*field_index=*/3,
/*field_name=*/"f3", FieldType::BOOLEAN,
Literal(true)),
}));
-
ASSERT_NOK_WITH_MSG(PredicateValidator::ValidatePredicateWithLiterals(predicate),
- "field f1 has field type BIGINT in literal,
mismatch "
- "field type FLOAT in predicate");
+ ASSERT_NOK_WITH_MSG(
+ PredicateValidator::ValidatePredicateWithSchema(*schema, predicate,
+
/*validate_field_idx=*/true),
+ "field f1 has field type BIGINT in literal, mismatch field type
FLOAT in predicate");
}
{
// f2 field type is STRING, literal type is BINARY
@@ -93,9 +107,10 @@ TEST(PredicateValidatorTest, TestValidateLiterals) {
PredicateBuilder::Equal(/*field_index=*/3,
/*field_name=*/"f3", FieldType::BOOLEAN,
Literal(true)),
}));
-
ASSERT_NOK_WITH_MSG(PredicateValidator::ValidatePredicateWithLiterals(predicate),
- "field f2 has field type BINARY in literal,
mismatch "
- "field type STRING in predicate");
+ ASSERT_NOK_WITH_MSG(
+ PredicateValidator::ValidatePredicateWithSchema(*schema, predicate,
+
/*validate_field_idx=*/true),
+ "field f2 has field type BINARY in literal, mismatch field type
STRING in predicate");
}
{
// f2 literal is null
@@ -111,8 +126,10 @@ TEST(PredicateValidatorTest, TestValidateLiterals) {
PredicateBuilder::Equal(/*field_index=*/3,
/*field_name=*/"f3", FieldType::BOOLEAN,
Literal(true)),
}));
-
ASSERT_NOK_WITH_MSG(PredicateValidator::ValidatePredicateWithLiterals(predicate),
- "literal cannot be null in predicate, field name
f2");
+ ASSERT_NOK_WITH_MSG(
+ PredicateValidator::ValidatePredicateWithSchema(*schema, predicate,
+
/*validate_field_idx=*/true),
+ "literal cannot be null in predicate, field name f2");
}
}
@@ -158,8 +175,7 @@ TEST(PredicateValidatorTest, TestValidateSchema) {
/*validate_field_idx=*/true));
}
{
- // f2 schema type is DECIMAL(23,5), predicate type can be different
precision and scale,
- // such as DECIMAL(22,4)
+ // f2 schema type is DECIMAL(23,5), but the literal scale is 4.
std::shared_ptr<arrow::Schema> schema =
arrow::schema(arrow::FieldVector({
arrow::field("f0", arrow::int16()),
arrow::field("f1", arrow::float32()),
@@ -179,9 +195,11 @@ TEST(PredicateValidatorTest, TestValidateSchema) {
PredicateBuilder::Equal(/*field_index=*/3,
/*field_name=*/"f3", FieldType::BOOLEAN,
Literal(true)),
}));
-
ASSERT_OK(PredicateValidator::ValidatePredicateWithLiterals(predicate));
- ASSERT_OK(PredicateValidator::ValidatePredicateWithSchema(*schema,
predicate,
-
/*validate_field_idx=*/true));
+ ASSERT_NOK_WITH_MSG(
+ PredicateValidator::ValidatePredicateWithSchema(*schema, predicate,
+
/*validate_field_idx=*/true),
+ "decimal literal for field f2 has scale 4, expected 5; rescale the
literal before "
+ "building the predicate");
}
{
// predicate field idx mismatch
@@ -341,4 +359,62 @@ TEST(PredicateValidatorTest, TestValidateSchema) {
"field f2 does not exist in schema");
}
}
+
+TEST(PredicateValidatorTest, TestValidateDecimalLiteral) {
+ std::shared_ptr<arrow::Schema> schema =
+ arrow::schema({arrow::field("amount", arrow::decimal128(10, 2))});
+
+ {
+ auto predicate =
+ PredicateBuilder::Equal(/*field_index=*/0,
/*field_name=*/"amount", FieldType::DECIMAL,
+ Literal(Decimal(10, 2, 12345)));
+ ASSERT_OK(PredicateValidator::ValidatePredicateWithSchema(*schema,
predicate,
+
/*validate_field_idx=*/true));
+ }
+ {
+ // Literal precision metadata may be smaller than the field precision.
+ auto predicate = PredicateBuilder::Equal(/*field_index=*/0,
/*field_name=*/"amount",
+ FieldType::DECIMAL,
Literal(Decimal(9, 2, 12345)));
+ ASSERT_OK(PredicateValidator::ValidatePredicateWithSchema(*schema,
predicate,
+
/*validate_field_idx=*/true));
+ }
+ {
+ // Literal precision metadata may be larger than the field precision
if the value fits.
+ auto predicate =
+ PredicateBuilder::Equal(/*field_index=*/0,
/*field_name=*/"amount", FieldType::DECIMAL,
+ Literal(Decimal(12, 2, 12345)));
+ ASSERT_OK(PredicateValidator::ValidatePredicateWithSchema(*schema,
predicate,
+
/*validate_field_idx=*/true));
+ }
+ {
+ auto predicate =
+ PredicateBuilder::Equal(/*field_index=*/0,
/*field_name=*/"amount", FieldType::DECIMAL,
+ Literal(Decimal(10, 1, 12345)));
+ ASSERT_NOK_WITH_MSG(
+ PredicateValidator::ValidatePredicateWithSchema(*schema, predicate,
+
/*validate_field_idx=*/true),
+ "decimal literal for field amount has scale 1, expected 2; rescale
the literal before "
+ "building the predicate");
+ }
+ {
+ auto predicate = PredicateBuilder::In(
+ /*field_index=*/0, /*field_name=*/"amount", FieldType::DECIMAL,
+ {Literal(Decimal(10, 2, 12345)), Literal(Decimal(10, 3, 123450))});
+ ASSERT_NOK_WITH_MSG(
+ PredicateValidator::ValidatePredicateWithSchema(*schema, predicate,
+
/*validate_field_idx=*/true),
+ "decimal literal for field amount has scale 3, expected 2; rescale
the literal before "
+ "building the predicate");
+ }
+ {
+ auto predicate =
+ PredicateBuilder::Equal(/*field_index=*/0,
/*field_name=*/"amount", FieldType::DECIMAL,
+ Literal(Decimal(10, 2, 10000000000LL)));
+ ASSERT_NOK_WITH_MSG(
+ PredicateValidator::ValidatePredicateWithSchema(*schema, predicate,
+
/*validate_field_idx=*/true),
+ "decimal literal 100000000.00 for field amount does not fit field
type DECIMAL(10, "
+ "2)");
+ }
+}
} // namespace paimon::test
diff --git a/src/paimon/core/operation/internal_read_context.cpp
b/src/paimon/core/operation/internal_read_context.cpp
index dd60ce40..c1a07eca 100644
--- a/src/paimon/core/operation/internal_read_context.cpp
+++ b/src/paimon/core/operation/internal_read_context.cpp
@@ -278,8 +278,6 @@ Result<std::unique_ptr<InternalReadContext>>
InternalReadContext::Create(
if (context->GetPredicate()) {
PAIMON_RETURN_NOT_OK(PredicateValidator::ValidatePredicateWithSchema(
*read_schema, context->GetPredicate(),
/*validate_field_idx=*/true));
- PAIMON_RETURN_NOT_OK(
-
PredicateValidator::ValidatePredicateWithLiterals(context->GetPredicate()));
}
if (!context->GetMemoryPool()) {
diff --git a/src/paimon/core/table/format/format_table_read.cpp
b/src/paimon/core/table/format/format_table_read.cpp
index 7bc14b59..86b1a880 100644
--- a/src/paimon/core/table/format/format_table_read.cpp
+++ b/src/paimon/core/table/format/format_table_read.cpp
@@ -425,7 +425,6 @@ Result<std::unique_ptr<FormatTableRead>>
FormatTableRead::CreateInternal(
// index is not among them: everything downstream resolves a field by
name.
PAIMON_RETURN_NOT_OK(PredicateValidator::ValidatePredicateWithSchema(
*impl->read_schema, predicate, /*validate_field_idx=*/false));
-
PAIMON_RETURN_NOT_OK(PredicateValidator::ValidatePredicateWithLiterals(predicate));
if (enable_predicate_filter) {
impl->filter_predicate = predicate;
}
diff --git a/src/paimon/core/table/source/table_scan.cpp
b/src/paimon/core/table/source/table_scan.cpp
index 20e13ce0..033a4e79 100644
--- a/src/paimon/core/table/source/table_scan.cpp
+++ b/src/paimon/core/table/source/table_scan.cpp
@@ -397,8 +397,6 @@ Result<std::unique_ptr<TableScan>> NewDataTableScan(
PAIMON_RETURN_NOT_OK(PredicateValidator::ValidatePredicateWithSchema(
*arrow_schema, context->GetScanFilters()->GetPredicate(),
/*validate_field_idx=*/false));
- PAIMON_RETURN_NOT_OK(PredicateValidator::ValidatePredicateWithLiterals(
- context->GetScanFilters()->GetPredicate()));
}
PAIMON_ASSIGN_OR_RAISE(std::vector<std::string> external_paths,
core_options.CreateExternalPaths());
diff --git a/test/inte/blob_table_inte_test.cpp
b/test/inte/blob_table_inte_test.cpp
index 4419b356..bfba944a 100644
--- a/test/inte/blob_table_inte_test.cpp
+++ b/test/inte/blob_table_inte_test.cpp
@@ -2225,9 +2225,8 @@ TEST_P(BlobTableInteTest, TestPartitionWithPredicate) {
}
{
// set partition predicate and data field predicate, blob type not
support predicate
- auto equal =
- PredicateBuilder::Equal(/*field_index=*/1, /*field_name=*/"f1",
FieldType::STRING,
- Literal(FieldType::BLOB, "2024", 4));
+ auto equal = PredicateBuilder::Equal(/*field_index=*/1,
/*field_name=*/"f1",
+ FieldType::BLOB,
Literal(FieldType::BLOB, "2024", 4));
auto greater_than = PredicateBuilder::GreaterThan(/*field_index=*/0,
/*field_name=*/"f0",
FieldType::INT,
Literal(100));
ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({equal,
greater_than}));
diff --git a/test/inte/global_index_test.cpp b/test/inte/global_index_test.cpp
index 517d55ac..355220c7 100644
--- a/test/inte/global_index_test.cpp
+++ b/test/inte/global_index_test.cpp
@@ -3394,6 +3394,14 @@ TEST_P(GlobalIndexTest,
TestBTreeScanWithPartitionWithMultiMeta) {
auto gt_mid,
reader->VisitGreaterThan(Literal(Decimal::FromUnscaledLong(10 *
123456L, 18, 6))));
ASSERT_EQ(count_rows(gt_mid), 18);
+
+ // Global index readers do not rescale Decimal literals. A
mathematically equivalent
+ // literal with a different scale produces an incorrect empty result,
so callers must use
+ // the field's scale.
+ ASSERT_OK_AND_ASSIGN(auto eq_same_value_different_scale,
+
reader->VisitEqual(Literal(Decimal::FromUnscaledLong(
+ 5 * 1234560L, /*precision=*/18,
/*scale=*/7))));
+ ASSERT_EQ(count_rows(eq_same_value_different_scale), 0);
}
// ---- col_string (values are "str_00000" .. "str_00019") ----
diff --git a/test/inte/read_inte_test.cpp b/test/inte/read_inte_test.cpp
index de93a035..4b2f0f21 100644
--- a/test/inte/read_inte_test.cpp
+++ b/test/inte/read_inte_test.cpp
@@ -2078,7 +2078,7 @@ TEST_P(ReadInteTest,
TestAppendReadWithComplexTypePredicate) {
PredicateBuilder::And(
{PredicateBuilder::Or(
{PredicateBuilder::GreaterThan(/*field_index=*/4,
/*field_name=*/"f5",
- FieldType::DECIMAL,
Literal(Decimal(5, 2, 0))),
+ FieldType::DECIMAL,
Literal(Decimal(23, 5, 0))),
PredicateBuilder::LessThan(/*field_index=*/2,
/*field_name=*/"f4",
FieldType::TIMESTAMP,
Literal(Timestamp(-2240521239999l, 1002))),
diff --git a/test/inte/scan_inte_test.cpp b/test/inte/scan_inte_test.cpp
index f626dba6..3f949484 100644
--- a/test/inte/scan_inte_test.cpp
+++ b/test/inte/scan_inte_test.cpp
@@ -1452,7 +1452,7 @@ TEST_P(ScanInteTest,
TestScanAppendComplexDataWithSnapshot4WithPredicateFilter)
Literal(paimon::Timestamp(1735344000, 0)));
auto predicate2 = PredicateBuilder::GreaterThan(
/*field_index=*/4, /*field_name=*/"f5", FieldType::DECIMAL,
- Literal(paimon::Decimal(5, 2, 0)));
+ Literal(paimon::Decimal(23, 5, 0)));
ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({predicate1,
predicate2}));
ScanContextBuilder context_builder(table_path);
