This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new fcb3a29c6a8 [feature](tde) add interface for table-level encryption
(#54995)
fcb3a29c6a8 is described below
commit fcb3a29c6a8a83da362b9ff59032afae91d96e16
Author: Luwei <[email protected]>
AuthorDate: Mon Aug 25 21:15:40 2025 +0800
[feature](tde) add interface for table-level encryption (#54995)
Co-authored-by: Siyang Tang <[email protected]>
---
be/src/cloud/cloud_tablet.cpp | 2 +
be/src/cloud/pb_convert.cpp | 12 +++++
be/src/io/CMakeLists.txt | 7 +++
.../fs/encrypted_fs_factory.cpp} | 45 ++++++------------
.../wal_writer.h => io/fs/encrypted_fs_factory.h} | 36 +++-----------
be/src/io/fs/file_writer.h | 1 +
be/src/olap/delta_writer_v2.cpp | 1 +
be/src/olap/rowset/rowset_meta.cpp | 43 +++++++++++++----
be/src/olap/rowset/rowset_meta.h | 4 ++
be/src/olap/rowset/rowset_writer_context.h | 52 +++++++++++++++------
be/src/olap/tablet.cpp | 2 +
be/src/olap/tablet_meta.cpp | 24 +++++++++-
be/src/olap/tablet_meta.h | 8 +++-
be/src/olap/task/index_builder.cpp | 2 +-
be/src/olap/wal/wal_reader.cpp | 26 +++++++++--
be/src/olap/wal/wal_writer.cpp | 50 ++++++++++++++++++--
be/src/olap/wal/wal_writer.h | 7 ++-
be/src/runtime/exec_env.h | 3 ++
be/src/util/url_coding.cpp | 9 ++--
be/src/vec/sink/writer/vwal_writer.cpp | 13 ++++--
be/src/vec/sink/writer/vwal_writer.h | 5 +-
be/test/exprs/bloom_filter_func_test.cpp | 5 +-
be/test/olap/wal/wal_reader_writer_test.cpp | 6 +--
build.sh | 6 +++
.../main/java/org/apache/doris/common/Config.java | 30 ++++++++++++
.../org/apache/doris/alter/CloudRollupJobV2.java | 2 +-
.../apache/doris/alter/CloudSchemaChangeJobV2.java | 2 +-
.../java/org/apache/doris/alter/RollupJobV2.java | 2 +-
.../org/apache/doris/alter/SchemaChangeJobV2.java | 2 +-
.../java/org/apache/doris/backup/RestoreJob.java | 2 +-
.../java/org/apache/doris/catalog/OlapTable.java | 16 +++++++
.../org/apache/doris/catalog/TableProperty.java | 26 ++++++++++-
.../apache/doris/cloud/backup/CloudRestoreJob.java | 2 +-
.../cloud/datasource/CloudInternalCatalog.java | 7 ++-
.../apache/doris/common/util/PropertyAnalyzer.java | 32 +++++++++++++
.../apache/doris/datasource/InternalCatalog.java | 20 +++++++-
.../org/apache/doris/master/ReportHandler.java | 2 +-
.../apache/doris/service/FrontendServiceImpl.java | 45 ++++++++++++++++++
.../org/apache/doris/task/CreateReplicaTask.java | 7 ++-
.../java/org/apache/doris/task/AgentTaskTest.java | 3 +-
gensrc/proto/olap_file.proto | 2 +
gensrc/thrift/AgentService.thrift | 1 +
gensrc/thrift/FrontendService.thrift | 2 +
.../data/query_p0/system/test_table_properties.out | Bin 16201 -> 16633 bytes
44 files changed, 449 insertions(+), 125 deletions(-)
diff --git a/be/src/cloud/cloud_tablet.cpp b/be/src/cloud/cloud_tablet.cpp
index b40ece24792..630a53ce562 100644
--- a/be/src/cloud/cloud_tablet.cpp
+++ b/be/src/cloud/cloud_tablet.cpp
@@ -733,6 +733,7 @@ Result<std::unique_ptr<RowsetWriter>>
CloudTablet::create_rowset_writer(
context.index_id = index_id();
context.partition_id = partition_id();
context.enable_unique_key_merge_on_write =
enable_unique_key_merge_on_write();
+ context.encrypt_algorithm = tablet_meta()->encryption_algorithm();
return RowsetFactory::create_rowset_writer(_engine, context, vertical);
}
@@ -774,6 +775,7 @@ Result<std::unique_ptr<RowsetWriter>>
CloudTablet::create_transient_rowset_write
context.partition_id = partition_id();
context.enable_unique_key_merge_on_write =
enable_unique_key_merge_on_write();
context.txn_expiration = txn_expiration;
+ context.encrypt_algorithm = tablet_meta()->encryption_algorithm();
auto storage_resource = rowset.rowset_meta()->remote_storage_resource();
if (!storage_resource) {
diff --git a/be/src/cloud/pb_convert.cpp b/be/src/cloud/pb_convert.cpp
index c220820c6dc..b4319cdb946 100644
--- a/be/src/cloud/pb_convert.cpp
+++ b/be/src/cloud/pb_convert.cpp
@@ -547,6 +547,9 @@ void doris_tablet_meta_to_cloud(TabletMetaCloudPB* out,
const TabletMetaPB& in)
if (in.has_schema_version()) {
out->set_schema_version(in.schema_version());
}
+ if (in.has_encryption_algorithm()) {
+ out->set_encryption_algorithm(in.encryption_algorithm());
+ }
}
void doris_tablet_meta_to_cloud(TabletMetaCloudPB* out, TabletMetaPB&& in) {
@@ -621,6 +624,9 @@ void doris_tablet_meta_to_cloud(TabletMetaCloudPB* out,
TabletMetaPB&& in) {
if (in.has_schema_version()) {
out->set_schema_version(in.schema_version());
}
+ if (in.has_encryption_algorithm()) {
+ out->set_encryption_algorithm(in.encryption_algorithm());
+ }
}
TabletMetaPB cloud_tablet_meta_to_doris(const TabletMetaCloudPB& in) {
@@ -702,6 +708,9 @@ void cloud_tablet_meta_to_doris(TabletMetaPB* out, const
TabletMetaCloudPB& in)
if (in.has_schema_version()) {
out->set_schema_version(in.schema_version());
}
+ if (in.has_encryption_algorithm()) {
+ out->set_encryption_algorithm(in.encryption_algorithm());
+ }
}
void cloud_tablet_meta_to_doris(TabletMetaPB* out, TabletMetaCloudPB&& in) {
@@ -776,6 +785,9 @@ void cloud_tablet_meta_to_doris(TabletMetaPB* out,
TabletMetaCloudPB&& in) {
if (in.has_schema_version()) {
out->set_schema_version(in.schema_version());
}
+ if (in.has_encryption_algorithm()) {
+ out->set_encryption_algorithm(in.encryption_algorithm());
+ }
}
#include "common/compile_check_end.h"
diff --git a/be/src/io/CMakeLists.txt b/be/src/io/CMakeLists.txt
index 02b34f2f0ea..dbd52e3560d 100644
--- a/be/src/io/CMakeLists.txt
+++ b/be/src/io/CMakeLists.txt
@@ -25,6 +25,13 @@ file(GLOB_RECURSE IO_FILES CONFIGURE_DEPENDS *.cpp)
if(BUILD_AZURE STREQUAL "OFF")
list(REMOVE_ITEM IO_FILES
"${CMAKE_CURRENT_SOURCE_DIR}/fs/azure_obj_storage_client.cpp")
endif()
+
+if(NOT "${WITH_TDE_DIR}" STREQUAL "")
+ list(REMOVE_ITEM IO_FILES
"${CMAKE_CURRENT_SOURCE_DIR}/fs/encrypted_fs_factory.cpp")
+ file(GLOB_RECURSE EXTRA_SOURCES
"${CMAKE_CURRENT_SOURCE_DIR}/../${WITH_TDE_DIR}/*.cpp")
+ list(APPEND IO_FILES ${EXTRA_SOURCES})
+endif()
+
list(REMOVE_ITEM IO_FILES
"${CMAKE_CURRENT_SOURCE_DIR}/fs/benchmark/fs_benchmark_tool.cpp")
add_library(IO STATIC ${IO_FILES})
diff --git a/be/src/olap/wal/wal_writer.h
b/be/src/io/fs/encrypted_fs_factory.cpp
similarity index 52%
copy from be/src/olap/wal/wal_writer.h
copy to be/src/io/fs/encrypted_fs_factory.cpp
index f730e026660..4f18c9b9c1c 100644
--- a/be/src/olap/wal/wal_writer.h
+++ b/be/src/io/fs/encrypted_fs_factory.cpp
@@ -15,39 +15,22 @@
// specific language governing permissions and limitations
// under the License.
-#pragma once
+#include "io/fs/encrypted_fs_factory.h"
-#include "common/status.h"
-#include "gen_cpp/internal_service.pb.h"
-#include "io/fs/file_reader_writer_fwd.h"
-
-namespace doris {
-
-using PBlockArray = std::vector<PBlock*>;
-extern const char* k_wal_magic;
-extern const uint32_t k_wal_magic_length;
-
-class WalWriter {
-public:
- explicit WalWriter(const std::string& file_name);
- ~WalWriter();
+#include <gen_cpp/olap_file.pb.h>
- Status init();
- Status finalize();
-
- Status append_blocks(const PBlockArray& blocks);
- Status append_header(std::string col_ids);
-
- std::string file_name() { return _file_name; };
+#include "common/exception.h"
+#include "common/status.h"
+#include "io/fs/file_system.h"
-public:
- static const int64_t LENGTH_SIZE = 8;
- static const int64_t CHECKSUM_SIZE = 4;
- static const int64_t VERSION_SIZE = 4;
+namespace doris::io {
-private:
- std::string _file_name;
- io::FileWriterPtr _file_writer;
-};
+FileSystemSPtr make_file_system(const FileSystemSPtr& inner,
EncryptionAlgorithmPB algorithm) {
+ if (algorithm == EncryptionAlgorithmPB::PLAINTEXT) {
+ return inner;
+ }
+ throw doris::Exception(ErrorCode::NOT_IMPLEMENTED_ERROR,
+ "Current version does not support TDE");
+}
-} // namespace doris
\ No newline at end of file
+} // namespace doris::io
diff --git a/be/src/olap/wal/wal_writer.h b/be/src/io/fs/encrypted_fs_factory.h
similarity index 51%
copy from be/src/olap/wal/wal_writer.h
copy to be/src/io/fs/encrypted_fs_factory.h
index f730e026660..3cf24b605d1 100644
--- a/be/src/olap/wal/wal_writer.h
+++ b/be/src/io/fs/encrypted_fs_factory.h
@@ -17,37 +17,15 @@
#pragma once
-#include "common/status.h"
-#include "gen_cpp/internal_service.pb.h"
-#include "io/fs/file_reader_writer_fwd.h"
+#include <gen_cpp/olap_file.pb.h>
-namespace doris {
+#include <memory>
-using PBlockArray = std::vector<PBlock*>;
-extern const char* k_wal_magic;
-extern const uint32_t k_wal_magic_length;
+#include "io/fs/file_system.h"
+namespace doris::io {
-class WalWriter {
-public:
- explicit WalWriter(const std::string& file_name);
- ~WalWriter();
+struct EncryptionInfo;
- Status init();
- Status finalize();
+FileSystemSPtr make_file_system(const FileSystemSPtr& inner,
EncryptionAlgorithmPB algorithm);
- Status append_blocks(const PBlockArray& blocks);
- Status append_header(std::string col_ids);
-
- std::string file_name() { return _file_name; };
-
-public:
- static const int64_t LENGTH_SIZE = 8;
- static const int64_t CHECKSUM_SIZE = 4;
- static const int64_t VERSION_SIZE = 4;
-
-private:
- std::string _file_name;
- io::FileWriterPtr _file_writer;
-};
-
-} // namespace doris
\ No newline at end of file
+} // namespace doris::io
diff --git a/be/src/io/fs/file_writer.h b/be/src/io/fs/file_writer.h
index c547932f9d6..6220d1c9210 100644
--- a/be/src/io/fs/file_writer.h
+++ b/be/src/io/fs/file_writer.h
@@ -32,6 +32,7 @@
namespace doris::io {
class FileSystem;
struct FileCacheAllocatorBuilder;
+struct EncryptionInfo;
// Only affects remote file writers
struct FileWriterOptions {
diff --git a/be/src/olap/delta_writer_v2.cpp b/be/src/olap/delta_writer_v2.cpp
index c6a3ff0d697..ada1ccba5a1 100644
--- a/be/src/olap/delta_writer_v2.cpp
+++ b/be/src/olap/delta_writer_v2.cpp
@@ -123,6 +123,7 @@ Status DeltaWriterV2::init() {
context.data_dir = nullptr;
context.partial_update_info = _partial_update_info;
context.memtable_on_sink_support_index_v2 = true;
+ context.encrypt_algorithm = EncryptionAlgorithmPB::PLAINTEXT;
_rowset_writer = std::make_shared<BetaRowsetWriterV2>(_streams);
RETURN_IF_ERROR(_rowset_writer->init(context));
diff --git a/be/src/olap/rowset/rowset_meta.cpp
b/be/src/olap/rowset/rowset_meta.cpp
index 2ded5f3264b..0ed13789871 100644
--- a/be/src/olap/rowset/rowset_meta.cpp
+++ b/be/src/olap/rowset/rowset_meta.cpp
@@ -18,23 +18,29 @@
#include "olap/rowset/rowset_meta.h"
#include <gen_cpp/olap_file.pb.h>
+#include <glog/logging.h>
#include <memory>
#include <random>
#include "cloud/cloud_storage_engine.h"
#include "common/logging.h"
+#include "common/status.h"
#include "google/protobuf/util/message_differencer.h"
+#include "io/fs/encrypted_fs_factory.h"
+#include "io/fs/file_system.h"
#include "io/fs/file_writer.h"
#include "io/fs/local_file_system.h"
#include "json2pb/json_to_pb.h"
#include "json2pb/pb_to_json.h"
+#include "olap/base_tablet.h"
#include "olap/lru_cache.h"
#include "olap/olap_common.h"
#include "olap/storage_policy.h"
#include "olap/tablet_fwd.h"
#include "olap/tablet_schema.h"
#include "olap/tablet_schema_cache.h"
+#include "runtime/exec_env.h"
#include "vec/common/schema_util.h"
namespace doris {
@@ -94,17 +100,38 @@ bool RowsetMeta::json_rowset_meta(std::string*
json_rowset_meta) {
}
io::FileSystemSPtr RowsetMeta::fs() {
- if (is_local()) {
- return io::global_local_filesystem();
- }
+ auto fs = [this]() -> io::FileSystemSPtr {
+ if (is_local()) {
+ return io::global_local_filesystem();
+ }
- auto storage_resource = remote_storage_resource();
- if (storage_resource) {
- return storage_resource.value()->fs;
- } else {
- LOG(WARNING) << storage_resource.error();
+ auto storage_resource = remote_storage_resource();
+ if (storage_resource) {
+ return storage_resource.value()->fs;
+ } else {
+ LOG(WARNING) << storage_resource.error();
+ return nullptr;
+ }
+ }();
+
+#ifndef BE_TEST
+ auto algorithm = _determine_encryption_once.call([this]() ->
Result<EncryptionAlgorithmPB> {
+ auto maybe_tablet = ExecEnv::get_tablet(tablet_id());
+ if (!maybe_tablet) {
+ LOG(WARNING) << "get tablet failed: " << maybe_tablet.error();
+ return ResultError(maybe_tablet.error());
+ }
+ auto tablet = maybe_tablet.value();
+ return tablet->tablet_meta()->encryption_algorithm();
+ });
+ if (!algorithm.has_value()) {
+ // TODO: return a Result<FileSystemSPtr> in this method?
return nullptr;
}
+ return io::make_file_system(fs, algorithm.value());
+#else
+ return fs;
+#endif
}
Result<const StorageResource*> RowsetMeta::remote_storage_resource() {
diff --git a/be/src/olap/rowset/rowset_meta.h b/be/src/olap/rowset/rowset_meta.h
index 1d1c59ed569..6d4223f236d 100644
--- a/be/src/olap/rowset/rowset_meta.h
+++ b/be/src/olap/rowset/rowset_meta.h
@@ -27,6 +27,8 @@
#include "common/cast_set.h"
#include "common/config.h"
+#include "common/status.h"
+#include "io/fs/encrypted_fs_factory.h"
#include "io/fs/file_system.h"
#include "olap/metadata_adder.h"
#include "olap/olap_common.h"
@@ -34,6 +36,7 @@
#include "olap/storage_policy.h"
#include "olap/tablet_fwd.h"
#include "runtime/memory/lru_cache_policy.h"
+#include "util/once.h"
namespace doris {
@@ -409,6 +412,7 @@ private:
RowsetId _rowset_id;
StorageResource _storage_resource;
bool _is_removed_from_rowset_meta = false;
+ DorisCallOnce<Result<EncryptionAlgorithmPB>> _determine_encryption_once;
};
#include "common/compile_check_end.h"
diff --git a/be/src/olap/rowset/rowset_writer_context.h
b/be/src/olap/rowset/rowset_writer_context.h
index 97984b1fa15..ebc6be93d48 100644
--- a/be/src/olap/rowset/rowset_writer_context.h
+++ b/be/src/olap/rowset/rowset_writer_context.h
@@ -18,12 +18,21 @@
#pragma once
#include <gen_cpp/olap_file.pb.h>
+#include <glog/logging.h>
+#include <functional>
+#include <optional>
+
+#include "common/status.h"
+#include "io/fs/encrypted_fs_factory.h"
+#include "io/fs/file_system.h"
+#include "io/fs/file_writer.h"
#include "olap/olap_define.h"
#include "olap/partial_update_info.h"
#include "olap/storage_policy.h"
#include "olap/tablet.h"
#include "olap/tablet_schema.h"
+#include "runtime/exec_env.h"
namespace doris {
@@ -121,6 +130,8 @@ struct RowsetWriterContext {
// For remote rowset
std::optional<StorageResource> storage_resource;
+ std::optional<EncryptionAlgorithmPB> encrypt_algorithm;
+
bool is_local_rowset() const { return !storage_resource; }
std::string segment_path(int seg_id) const {
@@ -131,31 +142,42 @@ struct RowsetWriterContext {
}
}
- io::FileSystemSPtr fs() const {
- if (is_local_rowset()) {
- return io::global_local_filesystem();
- } else {
- return storage_resource->fs;
+ io::FileSystemSPtr fs() {
+ auto fs = [this]() -> io::FileSystemSPtr {
+ if (is_local_rowset()) {
+ return io::global_local_filesystem();
+ } else {
+ return storage_resource->fs;
+ }
+ }();
+
+ if (!encrypt_algorithm.has_value()) {
+#ifndef BE_TEST
+ constexpr std::string_view msg =
+ "RowsetWriterContext::determine_encryption is not called
when creating this "
+ "RowsetWriterContext, it will result in encrypted rowsets
left unencrypted";
+ auto st = Status::InternalError(msg);
+
+ LOG(WARNING) << st;
+ DCHECK(false) << st;
+#else
+ encrypt_algorithm = EncryptionAlgorithmPB::PLAINTEXT;
+#endif
+ return fs;
}
+ return io::make_file_system(fs, encrypt_algorithm.value());
}
- io::FileSystem& fs_ref() const {
- if (is_local_rowset()) {
- return *io::global_local_filesystem();
- } else {
- return *storage_resource->fs;
- }
- }
+ io::FileSystem& fs_ref() { return *fs(); }
- io::FileWriterOptions get_file_writer_options() const {
+ io::FileWriterOptions get_file_writer_options() {
io::FileWriterOptions opts {
.write_file_cache = write_file_cache,
.is_cold_data = is_hot_data,
.file_cache_expiration = file_cache_ttl_sec > 0 &&
newest_write_timestamp > 0
? newest_write_timestamp +
file_cache_ttl_sec
: 0,
- .approximate_bytes_to_write = approximate_bytes_to_write,
- };
+ .approximate_bytes_to_write = approximate_bytes_to_write};
return opts;
}
diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp
index e165963ffe9..758e1168274 100644
--- a/be/src/olap/tablet.cpp
+++ b/be/src/olap/tablet.cpp
@@ -2038,6 +2038,8 @@ void
Tablet::_init_context_common_fields(RowsetWriterContext& context) {
context.data_dir = data_dir();
context.enable_unique_key_merge_on_write =
enable_unique_key_merge_on_write();
+
+ context.encrypt_algorithm = tablet_meta()->encryption_algorithm();
}
Status Tablet::create_rowset(const RowsetMetaSharedPtr& rowset_meta,
RowsetSharedPtr* rowset) {
diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp
index d6759fda558..028e9432cc9 100644
--- a/be/src/olap/tablet_meta.cpp
+++ b/be/src/olap/tablet_meta.cpp
@@ -18,6 +18,7 @@
#include "olap/tablet_meta.h"
#include <gen_cpp/Descriptors_types.h>
+#include <gen_cpp/FrontendService_types.h>
#include <gen_cpp/Types_types.h>
#include <gen_cpp/olap_common.pb.h>
#include <gen_cpp/olap_file.pb.h>
@@ -101,7 +102,8 @@ TabletMetaSharedPtr TabletMeta::create(
request.time_series_compaction_file_count_threshold,
request.time_series_compaction_time_threshold_seconds,
request.time_series_compaction_empty_rowsets_threshold,
- request.time_series_compaction_level_threshold,
inverted_index_file_storage_format);
+ request.time_series_compaction_level_threshold,
inverted_index_file_storage_format,
+ request.tde_algorithm);
}
TabletMeta::~TabletMeta() {
@@ -128,7 +130,8 @@ TabletMeta::TabletMeta(int64_t table_id, int64_t
partition_id, int64_t tablet_id
int64_t time_series_compaction_time_threshold_seconds,
int64_t time_series_compaction_empty_rowsets_threshold,
int64_t time_series_compaction_level_threshold,
- TInvertedIndexFileStorageFormat::type
inverted_index_file_storage_format)
+ TInvertedIndexFileStorageFormat::type
inverted_index_file_storage_format,
+ TEncryptionAlgorithm::type tde_algorithm)
: _tablet_uid(0, 0),
_schema(new TabletSchema),
_delete_bitmap(new DeleteBitmap(tablet_id)) {
@@ -364,6 +367,17 @@ TabletMeta::TabletMeta(int64_t table_id, int64_t
partition_id, int64_t tablet_id
tmp_binlog_config.to_pb(tablet_meta_pb.mutable_binlog_config());
}
+ switch (tde_algorithm) {
+ case doris::TEncryptionAlgorithm::AES256:
+
tablet_meta_pb.set_encryption_algorithm(EncryptionAlgorithmPB::AES_256_CTR);
+ break;
+ case doris::TEncryptionAlgorithm::SM4:
+
tablet_meta_pb.set_encryption_algorithm(EncryptionAlgorithmPB::SM4_128_CTR);
+ break;
+ default:
+
tablet_meta_pb.set_encryption_algorithm(EncryptionAlgorithmPB::PLAINTEXT);
+ }
+
init_from_pb(tablet_meta_pb);
}
@@ -772,6 +786,10 @@ void TabletMeta::init_from_pb(const TabletMetaPB&
tablet_meta_pb) {
tablet_meta_pb.time_series_compaction_empty_rowsets_threshold();
_time_series_compaction_level_threshold =
tablet_meta_pb.time_series_compaction_level_threshold();
+
+ if (tablet_meta_pb.has_encryption_algorithm()) {
+ _encryption_algorithm = tablet_meta_pb.encryption_algorithm();
+ }
}
void TabletMeta::to_meta_pb(TabletMetaPB* tablet_meta_pb) {
@@ -863,6 +881,8 @@ void TabletMeta::to_meta_pb(TabletMetaPB* tablet_meta_pb) {
time_series_compaction_empty_rowsets_threshold());
tablet_meta_pb->set_time_series_compaction_level_threshold(
time_series_compaction_level_threshold());
+
+ tablet_meta_pb->set_encryption_algorithm(_encryption_algorithm);
}
void TabletMeta::to_json(string* json_string, json2pb::Pb2JsonOptions&
options) {
diff --git a/be/src/olap/tablet_meta.h b/be/src/olap/tablet_meta.h
index 8627e2fabc0..cce0afcd560 100644
--- a/be/src/olap/tablet_meta.h
+++ b/be/src/olap/tablet_meta.h
@@ -18,6 +18,7 @@
#pragma once
#include <gen_cpp/AgentService_types.h>
+#include <gen_cpp/FrontendService_types.h>
#include <gen_cpp/olap_file.pb.h>
#include <stdint.h>
@@ -115,7 +116,8 @@ public:
int64_t time_series_compaction_empty_rowsets_threshold = 5,
int64_t time_series_compaction_level_threshold = 1,
TInvertedIndexFileStorageFormat::type
inverted_index_file_storage_format =
- TInvertedIndexFileStorageFormat::V2);
+ TInvertedIndexFileStorageFormat::V2,
+ TEncryptionAlgorithm::type tde_algorithm =
TEncryptionAlgorithm::PLAINTEXT);
// If need add a filed in TableMeta, filed init copy in copy construct
function
TabletMeta(const TabletMeta& tablet_meta);
TabletMeta(TabletMeta&& tablet_meta) = delete;
@@ -301,6 +303,8 @@ public:
int64_t avg_rs_meta_serialize_size() const { return
_avg_rs_meta_serialize_size; }
+ EncryptionAlgorithmPB encryption_algorithm() const { return
_encryption_algorithm; }
+
private:
Status _save_meta(DataDir* data_dir);
void _check_mow_rowset_cache_version_size(size_t
rowset_cache_version_size);
@@ -363,6 +367,8 @@ private:
// cloud
int64_t _ttl_seconds = 0;
+ EncryptionAlgorithmPB _encryption_algorithm = PLAINTEXT;
+
mutable std::shared_mutex _meta_lock;
};
diff --git a/be/src/olap/task/index_builder.cpp
b/be/src/olap/task/index_builder.cpp
index 0442da84aa3..00cc8dd0d90 100644
--- a/be/src/olap/task/index_builder.cpp
+++ b/be/src/olap/task/index_builder.cpp
@@ -357,7 +357,7 @@ Status
IndexBuilder::handle_single_rowset(RowsetMetaSharedPtr output_rowset_meta
return Status::OK();
} else {
// create inverted index writer
- const auto& fs = io::global_local_filesystem();
+ const auto& fs = output_rowset_meta->fs();
auto output_rowset_schema = output_rowset_meta->tablet_schema();
size_t inverted_index_size = 0;
for (auto& seg_ptr : segments) {
diff --git a/be/src/olap/wal/wal_reader.cpp b/be/src/olap/wal/wal_reader.cpp
index 2e432134749..ded3adcd78f 100644
--- a/be/src/olap/wal/wal_reader.cpp
+++ b/be/src/olap/wal/wal_reader.cpp
@@ -17,12 +17,16 @@
#include "olap/wal/wal_reader.h"
+#include <string_view>
+#include <utility>
+
#include "common/status.h"
#include "io/fs/file_reader.h"
-#include "io/fs/local_file_system.h"
+#include "io/fs/file_system.h"
#include "io/fs/path.h"
#include "util/coding.h"
#include "util/crc32c.h"
+#include "util/string_util.h"
#include "wal_writer.h"
namespace doris {
@@ -30,7 +34,6 @@ namespace doris {
WalReader::WalReader(const std::string& file_name) : _file_name(file_name),
_offset(0) {}
WalReader::~WalReader() = default;
-
Status WalReader::_deserialize(PBlock& block, const std::string& buf, size_t
block_len,
size_t bytes_read) {
if (UNLIKELY(!block.ParseFromString(buf))) {
@@ -42,14 +45,29 @@ Status WalReader::_deserialize(PBlock& block, const
std::string& buf, size_t blo
return Status::OK();
}
+std::pair<int64_t, int64_t> parse_db_tb_from_wal_path(const std::string&
wal_path) {
+ auto ret = split(wal_path, "/");
+ DCHECK_GT(ret.size(), 3);
+ auto db_id_pos = ret.size() - 1 - 2;
+ auto tb_id_pos = ret.size() - 1 - 1;
+ auto db_id = std::stoll(ret[db_id_pos]);
+ auto tb_id = std::stoll(ret[tb_id_pos]);
+
+ return {db_id, tb_id};
+}
+
Status WalReader::init() {
+ auto [db_id, tb_id] = parse_db_tb_from_wal_path(_file_name);
+ io::FileSystemSPtr fs;
+ RETURN_IF_ERROR(determine_wal_fs(db_id, tb_id, fs));
bool exists = false;
- RETURN_IF_ERROR(io::global_local_filesystem()->exists(_file_name,
&exists));
+ RETURN_IF_ERROR(fs->exists(_file_name, &exists));
if (!exists) {
LOG(WARNING) << "not exist wal= " << _file_name;
return Status::NotFound("wal {} doesn't exist", _file_name);
}
- RETURN_IF_ERROR(io::global_local_filesystem()->open_file(_file_name,
&file_reader));
+ RETURN_IF_ERROR(fs->open_file(_file_name, &file_reader));
+
return Status::OK();
}
diff --git a/be/src/olap/wal/wal_writer.cpp b/be/src/olap/wal/wal_writer.cpp
index 5658eded564..b5151217cc2 100644
--- a/be/src/olap/wal/wal_writer.cpp
+++ b/be/src/olap/wal/wal_writer.cpp
@@ -17,13 +17,20 @@
#include "olap/wal/wal_writer.h"
+#include <gen_cpp/AgentService_types.h>
+#include <gen_cpp/FrontendService_types.h>
+
#include "common/config.h"
+#include "common/status.h"
+#include "io/fs/encrypted_fs_factory.h"
+#include "io/fs/file_system.h"
#include "io/fs/file_writer.h"
#include "io/fs/local_file_system.h"
#include "io/fs/path.h"
#include "olap/storage_engine.h"
#include "olap/wal/wal_manager.h"
#include "util/crc32c.h"
+#include "util/thrift_rpc_helper.h"
namespace doris {
@@ -34,15 +41,50 @@ WalWriter::WalWriter(const std::string& file_name) :
_file_name(file_name) {}
WalWriter::~WalWriter() {}
-Status WalWriter::init() {
+Status determine_wal_fs(int64_t db_id, int64_t tb_id, io::FileSystemSPtr& fs) {
+#ifndef BE_TEST
+ TNetworkAddress master_addr =
ExecEnv::GetInstance()->cluster_info()->master_fe_addr;
+ TGetTableTDEInfoRequest req;
+ req.__set_db_id(db_id);
+ req.__set_table_id(tb_id);
+ TGetTableTDEInfoResult ret;
+ RETURN_IF_ERROR(ThriftRpcHelper::rpc<FrontendServiceClient>(
+ master_addr.hostname, master_addr.port,
+ [&req, &ret](FrontendServiceConnection& client) {
+ client->getTableTDEInfo(ret, req);
+ }));
+ if (auto st = Status::create(ret.status); !st) {
+ return st;
+ }
+ auto encrypt_algorithm = [&ret]() -> EncryptionAlgorithmPB {
+ switch (ret.algorithm) {
+ case doris::TEncryptionAlgorithm::AES256:
+ return EncryptionAlgorithmPB::AES_256_CTR;
+ case doris::TEncryptionAlgorithm::SM4:
+ return EncryptionAlgorithmPB::SM4_128_CTR;
+ default:
+ return EncryptionAlgorithmPB::PLAINTEXT;
+ }
+ }();
+
+ auto local_fs = io::global_local_filesystem();
+ fs = io::make_file_system(local_fs, encrypt_algorithm);
+#else
+ fs = io::global_local_filesystem();
+#endif
+
+ return Status::OK();
+}
+
+Status WalWriter::init(const io::FileSystemSPtr& fs) {
io::Path wal_path = _file_name;
auto parent_path = wal_path.parent_path();
bool exists = false;
- RETURN_IF_ERROR(io::global_local_filesystem()->exists(parent_path,
&exists));
+ RETURN_IF_ERROR(fs->exists(parent_path, &exists));
if (!exists) {
-
RETURN_IF_ERROR(io::global_local_filesystem()->create_directory(parent_path));
+ RETURN_IF_ERROR(fs->create_directory(parent_path));
}
- RETURN_IF_ERROR(io::global_local_filesystem()->create_file(_file_name,
&_file_writer));
+ RETURN_IF_ERROR(fs->create_file(_file_name, &_file_writer));
LOG(INFO) << "create wal " << _file_name;
return Status::OK();
}
diff --git a/be/src/olap/wal/wal_writer.h b/be/src/olap/wal/wal_writer.h
index f730e026660..f4465517a59 100644
--- a/be/src/olap/wal/wal_writer.h
+++ b/be/src/olap/wal/wal_writer.h
@@ -20,6 +20,7 @@
#include "common/status.h"
#include "gen_cpp/internal_service.pb.h"
#include "io/fs/file_reader_writer_fwd.h"
+#include "io/fs/file_system.h"
namespace doris {
@@ -32,7 +33,7 @@ public:
explicit WalWriter(const std::string& file_name);
~WalWriter();
- Status init();
+ Status init(const io::FileSystemSPtr& fs);
Status finalize();
Status append_blocks(const PBlockArray& blocks);
@@ -50,4 +51,6 @@ private:
io::FileWriterPtr _file_writer;
};
-} // namespace doris
\ No newline at end of file
+Status determine_wal_fs(int64_t db_id, int64_t tb_id, io::FileSystemSPtr& fs);
+
+} // namespace doris
diff --git a/be/src/runtime/exec_env.h b/be/src/runtime/exec_env.h
index 72e96ed3027..e455af6fe8b 100644
--- a/be/src/runtime/exec_env.h
+++ b/be/src/runtime/exec_env.h
@@ -18,14 +18,17 @@
#pragma once
#include <common/multi_version.h>
+#include <gen_cpp/olap_file.pb.h>
#include <atomic>
#include <map>
#include <memory>
#include <mutex>
+#include <optional>
#include <string>
#include <vector>
+#include "common/config.h"
#include "common/status.h"
#include "io/cache/fs_file_cache_storage.h"
#include "olap/memtable_memory_limiter.h"
diff --git a/be/src/util/url_coding.cpp b/be/src/util/url_coding.cpp
index c442dbcb846..6af16f475b4 100644
--- a/be/src/util/url_coding.cpp
+++ b/be/src/util/url_coding.cpp
@@ -20,6 +20,7 @@
#include <curl/curl.h>
#include <libbase64.h>
+#include <cmath>
#include <sstream>
namespace doris {
@@ -82,9 +83,9 @@ bool url_decode(const std::string& in, std::string* out) {
}
void base64_encode(const std::string& in, std::string* out) {
- out->resize(size_t(in.length() * (4.0 / 3) + 1));
- auto len = base64_encode(reinterpret_cast<const unsigned
char*>(in.c_str()), in.length(),
- (unsigned char*)out->c_str());
+ out->resize((size_t)(4.0 * std::ceil(in.length() / 3.0)));
+ auto len = base64_encode(reinterpret_cast<const unsigned
char*>(in.data()), in.length(),
+ (unsigned char*)out->data());
out->resize(len);
}
@@ -115,7 +116,7 @@ int64_t base64_decode(const char* data, size_t length,
char* decoded_data) {
bool base64_decode(const std::string& in, std::string* out) {
out->resize(in.length());
- int64_t len = base64_decode(in.c_str(), in.length(), out->data());
+ int64_t len = base64_decode(in.data(), in.length(), out->data());
if (len < 0) {
return false;
}
diff --git a/be/src/vec/sink/writer/vwal_writer.cpp
b/be/src/vec/sink/writer/vwal_writer.cpp
index 76e0bf0679c..5a64b6f7c43 100644
--- a/be/src/vec/sink/writer/vwal_writer.cpp
+++ b/be/src/vec/sink/writer/vwal_writer.cpp
@@ -21,6 +21,8 @@
#include <sstream>
+#include "gen_cpp/FrontendService.h"
+#include "io/fs/encrypted_fs_factory.h"
#include "util/debug_points.h"
namespace doris {
@@ -40,6 +42,7 @@ VWalWriter::VWalWriter(int64_t db_id, int64_t tb_id, int64_t
wal_id,
VWalWriter::~VWalWriter() {}
Status VWalWriter::init() {
+ io::FileSystemSPtr wal_fs = io::global_local_filesystem();
#ifndef BE_TEST
if (config::group_commit_wait_replay_wal_finish) {
std::shared_ptr<std::mutex> lock = std::make_shared<std::mutex>();
@@ -49,8 +52,9 @@ Status VWalWriter::init() {
LOG(WARNING) << "fail to add wal_id " << _wal_id << " to
wal_cv_map";
}
}
+ RETURN_IF_ERROR(determine_wal_fs(_db_id, _tb_id, wal_fs));
#endif
- RETURN_IF_ERROR(_create_wal_writer(_wal_id, _wal_writer));
+ RETURN_IF_ERROR(_create_wal_writer(_wal_id, wal_fs, _wal_writer));
_wal_manager->add_wal_queue(_tb_id, _wal_id);
std::stringstream ss;
for (auto slot_desc : _slot_descs) {
@@ -90,12 +94,13 @@ Status VWalWriter::close() {
return Status::OK();
}
-Status VWalWriter::_create_wal_writer(int64_t wal_id,
std::shared_ptr<WalWriter>& wal_writer) {
+Status VWalWriter::_create_wal_writer(int64_t wal_id, const
io::FileSystemSPtr& fs,
+ std::shared_ptr<WalWriter>& wal_writer) {
std::string wal_path;
RETURN_IF_ERROR(_wal_manager->get_wal_path(wal_id, wal_path));
wal_writer = std::make_shared<WalWriter>(wal_path);
- RETURN_IF_ERROR(wal_writer->init());
+ RETURN_IF_ERROR(wal_writer->init(fs));
return Status::OK();
}
} // namespace vectorized
-} // namespace doris
\ No newline at end of file
+} // namespace doris
diff --git a/be/src/vec/sink/writer/vwal_writer.h
b/be/src/vec/sink/writer/vwal_writer.h
index a9fa218f330..19bd2761118 100644
--- a/be/src/vec/sink/writer/vwal_writer.h
+++ b/be/src/vec/sink/writer/vwal_writer.h
@@ -38,7 +38,8 @@ public:
Status close();
private:
- Status _create_wal_writer(int64_t wal_id, std::shared_ptr<WalWriter>&
wal_writer);
+ Status _create_wal_writer(int64_t wal_id, const io::FileSystemSPtr& fs,
+ std::shared_ptr<WalWriter>& wal_writer);
private:
int64_t _db_id;
@@ -51,4 +52,4 @@ private:
std::shared_ptr<WalWriter> _wal_writer;
};
} // namespace vectorized
-} // namespace doris
\ No newline at end of file
+} // namespace doris
diff --git a/be/test/exprs/bloom_filter_func_test.cpp
b/be/test/exprs/bloom_filter_func_test.cpp
index 5504d6b06f5..2b99fdef051 100644
--- a/be/test/exprs/bloom_filter_func_test.cpp
+++ b/be/test/exprs/bloom_filter_func_test.cpp
@@ -326,7 +326,8 @@ TEST_F(BloomFilterFuncTest, HashAlgorithm) {
"AAAAQAAAAIAACAAAAABAACAAAAAAQAAAAQAAAACAAAAAACAEAACAAgAACAQEAAAgAASAAAABAgAAEAAIDAAAAA"
"CQAQAAYgAAAAIBAiBAgAABAgBABAAICAAABCGAAIABBAAAAAAAIABAAAAACAAAAAAAABAAQAAAAAAAIBAAAAAA"
"AQAiABEAAQBAIgAAgBAQEEAAACACAQAABEgAAggAAQAAAUAQAAEQECCAAABAAIgHAAAACAEAAgAJQABAIAEAAA"
-
"gAAEAAAAAAEAAAAAAQAAABAAAQAAAAAEAAAAAEAACAEAAAAAUAAAAAIBAgCAAAQAAIAAAACBAIABAAAAAABg";
+
"gAAEAAAAAAEAAAAAAQAAABAAAQAAAAAEAAAAAEAACAEAAAAAUAAAAAIBAgCAAAQAAIAAAACBAIABAAAAAABg="
+ "=";
BloomFilterFunc<PrimitiveType::TYPE_INT> bloom_filter_func(false);
const size_t runtime_length = 1024;
RuntimeFilterParams params {1,
@@ -607,4 +608,4 @@ TEST_F(BloomFilterFuncTest, FindFixedLenOlapEngine) {
ASSERT_EQ(offsets2[1], 3);
}
-} // namespace doris
\ No newline at end of file
+} // namespace doris
diff --git a/be/test/olap/wal/wal_reader_writer_test.cpp
b/be/test/olap/wal/wal_reader_writer_test.cpp
index 5c318f82b4f..94fc8ff91a2 100644
--- a/be/test/olap/wal/wal_reader_writer_test.cpp
+++ b/be/test/olap/wal/wal_reader_writer_test.cpp
@@ -56,7 +56,7 @@ public:
static std::string _s_test_data_path;
};
-std::string WalReaderWriterTest::_s_test_data_path =
"./log/wal_reader_writer_test";
+std::string WalReaderWriterTest::_s_test_data_path =
"./log/wal_reader_writer_test/0/0";
size_t block_rows = 1024;
void covert_block_to_pb(
@@ -90,7 +90,7 @@ void generate_block(PBlock& pblock, int row_index) {
TEST_F(WalReaderWriterTest, TestWriteAndRead1) {
std::string file_name = _s_test_data_path + "/abcd123.txt";
auto wal_writer = WalWriter(file_name);
- static_cast<void>(wal_writer.init());
+ static_cast<void>(wal_writer.init(io::global_local_filesystem()));
size_t file_len = 0;
int64_t file_size = -1;
// add 1 block
@@ -138,4 +138,4 @@ TEST_F(WalReaderWriterTest, TestWriteAndRead1) {
static_cast<void>(wal_reader.finalize());
EXPECT_EQ(3, block_count);
}
-} // namespace doris
\ No newline at end of file
+} // namespace doris
diff --git a/build.sh b/build.sh
index 8bccbbe4030..98e9b671889 100755
--- a/build.sh
+++ b/build.sh
@@ -480,6 +480,10 @@ if [[ "${BUILD_BE_JAVA_EXTENSIONS}" -eq 1 &&
"${TARGET_SYSTEM}" == 'Darwin' ]];
fi
fi
+if [[ -z "${WITH_TDE_DIR}" ]]; then
+ WITH_TDE_DIR=''
+fi
+
echo "Get params:
BUILD_FE -- ${BUILD_FE}
BUILD_BE -- ${BUILD_BE}
@@ -506,6 +510,7 @@ echo "Get params:
DENABLE_CLANG_COVERAGE -- ${DENABLE_CLANG_COVERAGE}
DISPLAY_BUILD_TIME -- ${DISPLAY_BUILD_TIME}
ENABLE_PCH -- ${ENABLE_PCH}
+ WITH_TDE_DIR -- ${WITH_TDE_DIR}
"
# Clean and build generated code
@@ -612,6 +617,7 @@ if [[ "${BUILD_BE}" -eq 1 ]]; then
-DENABLE_CLANG_COVERAGE="${DENABLE_CLANG_COVERAGE}" \
-DDORIS_JAVA_HOME="${JAVA_HOME}" \
-DBUILD_AZURE="${BUILD_AZURE}" \
+ -DWITH_TDE_DIR="${WITH_TDE_DIR}" \
"${DORIS_HOME}/be"
if [[ "${OUTPUT_BE_BINARY}" -eq 1 ]]; then
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index 0ecd7fa477d..1fc6df750bd 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -3586,4 +3586,34 @@ public class Config extends ConfigBase {
@ConfField(mutable = true, masterOnly = true)
public static long create_partition_wait_seconds = 300;
+
+ @ConfField(mutable = true, description = {
+ "KMS 主密钥的 ID，用于生成和加密数据密钥",
+ "The ID of the master key in KMS, used for generating and encrypting
data keys"
+ })
+ public static String doris_tde_key_id = "";
+
+ @ConfField(mutable = true, description = {
+ "KMS 服务的访问地址（endpoint），需与密钥所在的 region 匹配",
+ "The endpoint of the KMS service, should match the region of the key"
+ })
+ public static String doris_tde_key_endpoint = "";
+
+ @ConfField(mutable = true, description = {
+ "KMS 密钥所属的区域，用于 SDK 调用时的区域配置",
+ "The region where the KMS key is located, used for SDK configuration"
+ })
+ public static String doris_tde_key_region = "";
+
+ @ConfField(mutable = true, description = {
+ "TDE（透明数据加密）的密钥提供方，目前支持 aws_kms",
+ "The key provider for TDE (Transparent Data Encryption), currently
supports aws_kms"
+ })
+ public static String doris_tde_key_provider = "";
+
+ @ConfField(mutable = true, description = {
+ "数据加密所使用的算法，默认 AES256，后续可能置空由 KMS 自动决定",
+ "The encryption algorithm used for data, default is AES256, may be set
to empty later for KMS to decide"
+ })
+ public static String doris_tde_algorithm = "PLAINTEXT";
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java
b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java
index d1f7c1af6c7..fbe85008752 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java
@@ -232,7 +232,7 @@ public class CloudRollupJobV2 extends RollupJobV2 {
null,
tbl.rowStorePageSize(),
tbl.variantEnableFlattenNested(), null,
- tbl.storagePageSize(),
+ tbl.storagePageSize(),
tbl.getTDEAlgorithmPB(),
tbl.storageDictPageSize(), true);
requestBuilder.addTabletMetas(builder);
} // end for rollupTablets
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java
b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java
index e4cd807479f..6e113eb5650 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java
@@ -254,7 +254,7 @@ public class CloudSchemaChangeJobV2 extends
SchemaChangeJobV2 {
tbl.getInvertedIndexFileStorageFormat(),
tbl.rowStorePageSize(),
tbl.variantEnableFlattenNested(),
clusterKeyUids,
- tbl.storagePageSize(),
+ tbl.storagePageSize(),
tbl.getTDEAlgorithmPB(),
tbl.storageDictPageSize(), true);
requestBuilder.addTabletMetas(builder);
} // end for rollupTablets
diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java
b/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java
index 9599b6f06a1..506cace3d9c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java
@@ -268,7 +268,7 @@ public class RollupJobV2 extends AlterJobV2 implements
GsonPostProcessable {
objectPool,
tbl.rowStorePageSize(),
tbl.variantEnableFlattenNested(),
- tbl.storagePageSize(),
+ tbl.storagePageSize(), tbl.getTDEAlgorithm(),
tbl.storageDictPageSize());
createReplicaTask.setBaseTablet(tabletIdMap.get(rollupTabletId),
baseSchemaHash);
if (this.storageFormat != null) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
index 3519aaf70e1..7bbcde4db06 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
@@ -323,7 +323,7 @@ public class SchemaChangeJobV2 extends AlterJobV2 {
objectPool,
tbl.rowStorePageSize(),
tbl.variantEnableFlattenNested(),
- tbl.storagePageSize(),
+ tbl.storagePageSize(),
tbl.getTDEAlgorithm(),
tbl.storageDictPageSize());
createReplicaTask.setBaseTablet(partitionIndexTabletMap.get(partitionId,
shadowIdxId)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
index b2355c3d159..8b70bb9a764 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
@@ -1440,7 +1440,7 @@ public class RestoreJob extends AbstractJob implements
GsonPostProcessable {
objectPool,
localTbl.rowStorePageSize(),
localTbl.variantEnableFlattenNested(),
- localTbl.storagePageSize(),
+ localTbl.storagePageSize(),
localTbl.getTDEAlgorithm(),
localTbl.storageDictPageSize());
task.setInvertedIndexFileStorageFormat(localTbl.getInvertedIndexFileStorageFormat());
task.setInRestoreMode(true);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
index 71816bd96d4..4762133d071 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
@@ -59,6 +59,7 @@ import
org.apache.doris.nereids.trees.plans.algebra.CatalogRelation;
import org.apache.doris.persist.ColocatePersistInfo;
import org.apache.doris.persist.gson.GsonPostProcessable;
import org.apache.doris.persist.gson.GsonUtils;
+import org.apache.doris.proto.OlapFile.EncryptionAlgorithmPB;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.OriginStatement;
import org.apache.doris.resource.Tag;
@@ -75,6 +76,7 @@ import org.apache.doris.system.BeSelectionPolicy;
import org.apache.doris.system.SystemInfoService;
import org.apache.doris.thrift.TColumn;
import org.apache.doris.thrift.TCompressionType;
+import org.apache.doris.thrift.TEncryptionAlgorithm;
import org.apache.doris.thrift.TFetchOption;
import org.apache.doris.thrift.TInvertedIndexFileStorageFormat;
import org.apache.doris.thrift.TNodeInfo;
@@ -2693,6 +2695,20 @@ public class OlapTable extends Table implements
MTMVRelatedTableIf, GsonPostProc
tableProperty.buildDataSortInfo();
}
+ public EncryptionAlgorithmPB getTDEAlgorithmPB() {
+ return tableProperty.getTDEAlgorithmPB();
+ }
+
+ public TEncryptionAlgorithm getTDEAlgorithm() {
+ return tableProperty.getTDEAlgorithm();
+ }
+
+ public void setEncryptionAlgorithm(TEncryptionAlgorithm algorithm) {
+ TableProperty tableProperty = getOrCreatTableProperty();
+
tableProperty.modifyTableProperties(PropertyAnalyzer.PROPERTIES_TDE_ALGORITHM,
algorithm.name());
+ tableProperty.buildTDEAlgorithm();
+ }
+
// return true if partition with given name already exist, both in
partitions and temp partitions.
// return false otherwise
public boolean checkPartitionNameExist(String partitionName) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
index cab70d5cf87..013bdb52173 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
@@ -22,7 +22,9 @@ import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.util.PropertyAnalyzer;
import org.apache.doris.persist.OperationType;
import org.apache.doris.persist.gson.GsonPostProcessable;
+import org.apache.doris.proto.OlapFile.EncryptionAlgorithmPB;
import org.apache.doris.thrift.TCompressionType;
+import org.apache.doris.thrift.TEncryptionAlgorithm;
import org.apache.doris.thrift.TInvertedIndexFileStorageFormat;
import org.apache.doris.thrift.TStorageFormat;
import org.apache.doris.thrift.TStorageMedium;
@@ -124,6 +126,7 @@ public class TableProperty implements GsonPostProcessable {
private String autoAnalyzePolicy =
PropertyAnalyzer.ENABLE_AUTO_ANALYZE_POLICY;
+ private TEncryptionAlgorithm encryptionAlgorithm =
TEncryptionAlgorithm.PLAINTEXT;
private DataSortInfo dataSortInfo = new DataSortInfo();
@@ -272,11 +275,31 @@ public class TableProperty implements GsonPostProcessable
{
return this;
}
+ public void buildTDEAlgorithm() {
+ String tdeAlgorithmName =
properties.getOrDefault(PropertyAnalyzer.PROPERTIES_TDE_ALGORITHM,
+ TEncryptionAlgorithm.PLAINTEXT.name());
+ encryptionAlgorithm = TEncryptionAlgorithm.valueOf(tdeAlgorithmName);
+ }
+
+ public TEncryptionAlgorithm getTDEAlgorithm() {
+ return encryptionAlgorithm;
+ }
+
+ public EncryptionAlgorithmPB getTDEAlgorithmPB() {
+ switch (encryptionAlgorithm) {
+ case AES256:
+ return EncryptionAlgorithmPB.AES_256_CTR;
+ case SM4:
+ return EncryptionAlgorithmPB.SM4_128_CTR;
+ default:
+ return EncryptionAlgorithmPB.PLAINTEXT;
+ }
+ }
+
public boolean disableAutoCompaction() {
return disableAutoCompaction;
}
-
public TableProperty buildVariantEnableFlattenNested() {
variantEnableFlattenNested = Boolean.parseBoolean(
properties.getOrDefault(PropertyAnalyzer.PROPERTIES_VARIANT_ENABLE_FLATTEN_NESTED,
"false"));
@@ -760,6 +783,7 @@ public class TableProperty implements GsonPostProcessable {
buildInAtomicRestore();
removeDuplicateReplicaNumProperty();
buildReplicaAllocation();
+ buildTDEAlgorithm();
}
// For some historical reason,
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/cloud/backup/CloudRestoreJob.java
b/fe/fe-core/src/main/java/org/apache/doris/cloud/backup/CloudRestoreJob.java
index 0e341d063a6..26d2f6b750a 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/cloud/backup/CloudRestoreJob.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/cloud/backup/CloudRestoreJob.java
@@ -387,7 +387,7 @@ public class CloudRestoreJob extends RestoreJob {
localTbl.getInvertedIndexFileStorageFormat(),
localTbl.rowStorePageSize(),
localTbl.variantEnableFlattenNested(),
clusterKeyUids,
- localTbl.storagePageSize(),
+ localTbl.storagePageSize(),
localTbl.getTDEAlgorithmPB(),
localTbl.storageDictPageSize(), false));
// In cloud mode all storage medium will be saved to HDD.
TabletMeta tabletMeta = new TabletMeta(db.getId(),
localTbl.getId(), restorePart.getId(),
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
index b3e352a5623..c1266f104d1 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
@@ -58,6 +58,7 @@ import org.apache.doris.common.MetaNotFoundException;
import org.apache.doris.datasource.InternalCatalog;
import org.apache.doris.proto.OlapCommon;
import org.apache.doris.proto.OlapFile;
+import org.apache.doris.proto.OlapFile.EncryptionAlgorithmPB;
import org.apache.doris.proto.Types;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.rpc.RpcException;
@@ -188,7 +189,7 @@ public class CloudInternalCatalog extends InternalCatalog {
effectiveIndexStorageFormat,
tbl.rowStorePageSize(),
tbl.variantEnableFlattenNested(), clusterKeyUids,
- tbl.storagePageSize(),
+ tbl.storagePageSize(), tbl.getTDEAlgorithmPB(),
tbl.storageDictPageSize(), true);
requestBuilder.addTabletMetas(builder);
}
@@ -222,7 +223,8 @@ public class CloudInternalCatalog extends InternalCatalog {
List<Integer> rowStoreColumnUniqueIds,
TInvertedIndexFileStorageFormat invertedIndexFileStorageFormat,
long pageSize,
boolean variantEnableFlattenNested, List<Integer> clusterKeyUids,
- long storagePageSize, long storageDictPageSize, boolean
createInitialRowset) throws DdlException {
+ long storagePageSize, EncryptionAlgorithmPB encryptionAlgorithm,
long storageDictPageSize,
+ boolean createInitialRowset) throws DdlException {
OlapFile.TabletMetaCloudPB.Builder builder =
OlapFile.TabletMetaCloudPB.newBuilder();
builder.setTableId(tableId);
builder.setIndexId(indexId);
@@ -378,6 +380,7 @@ public class CloudInternalCatalog extends InternalCatalog {
OlapFile.RowsetMetaCloudPB.Builder rowsetBuilder =
createInitialRowset(tablet, partitionId,
schemaHash, schema);
builder.addRsMetas(rowsetBuilder);
+ builder.setEncryptionAlgorithm(encryptionAlgorithm);
}
return builder;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
index 5ee0598e8e7..6ee9a62844e 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
@@ -44,6 +44,7 @@ import org.apache.doris.policy.StoragePolicy;
import org.apache.doris.resource.Tag;
import org.apache.doris.system.SystemInfoService;
import org.apache.doris.thrift.TCompressionType;
+import org.apache.doris.thrift.TEncryptionAlgorithm;
import org.apache.doris.thrift.TInvertedIndexFileStorageFormat;
import org.apache.doris.thrift.TSortType;
import org.apache.doris.thrift.TStorageFormat;
@@ -57,6 +58,7 @@ import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
+import lombok.val;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -250,6 +252,10 @@ public class PropertyAnalyzer {
public static final String PROPERTIES_VARIANT_MAX_SUBCOLUMNS_COUNT =
"variant_max_subcolumns_count";
public static final String PROPERTIES_VARIANT_ENABLE_TYPED_PATHS_TO_SPARSE
= "variant_enable_typed_paths_to_sparse";
+ public static final String PROPERTIES_TDE_ALGORITHM = "tde_algorithm";
+ public static final String AES256 = "AES256";
+ public static final String SM4 = "SM4";
+ public static final String PLAINTEXT = "PLAINTEXT";
public enum RewriteType {
PUT, // always put property
@@ -1868,4 +1874,30 @@ public class PropertyAnalyzer {
}
return enableTypedPathsToSparse;
}
+
+ public static TEncryptionAlgorithm analyzeTDEAlgorithm(Map<String, String>
properties) throws AnalysisException {
+ String name;
+ if (properties == null ||
!properties.containsKey(PROPERTIES_TDE_ALGORITHM)) {
+ if (Config.doris_tde_algorithm.isEmpty()) {
+ return TEncryptionAlgorithm.PLAINTEXT;
+ }
+ name = Config.doris_tde_algorithm;
+ } else if (!PLAINTEXT.equals(Config.doris_tde_algorithm)) {
+ throw new AnalysisException("Cannot create a table on encrypted
FE,"
+ + " please set Config.doris_tde_algorithm to PLAINTEXT");
+ } else {
+ name = properties.remove(PROPERTIES_TDE_ALGORITHM);
+ }
+
+ if (AES256.equalsIgnoreCase(name)) {
+ return TEncryptionAlgorithm.AES256;
+ }
+ if (SM4.equalsIgnoreCase(name)) {
+ return TEncryptionAlgorithm.SM4;
+ }
+ if (PLAINTEXT.equalsIgnoreCase(name)) {
+ return TEncryptionAlgorithm.PLAINTEXT;
+ }
+ throw new AnalysisException("Invalid tde algorithm: " + name + ", only
support AES256 and SM4 currently");
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
index 782f9ce15d6..ec646be7e58 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
@@ -152,6 +152,7 @@ import org.apache.doris.task.AgentTaskQueue;
import org.apache.doris.task.CreateReplicaTask;
import org.apache.doris.task.DropReplicaTask;
import org.apache.doris.thrift.TCompressionType;
+import org.apache.doris.thrift.TEncryptionAlgorithm;
import org.apache.doris.thrift.TInvertedIndexFileStorageFormat;
import org.apache.doris.thrift.TStatusCode;
import org.apache.doris.thrift.TStorageFormat;
@@ -1656,6 +1657,9 @@ public class InternalCatalog implements
CatalogIf<Database> {
if
(!properties.containsKey(PropertyAnalyzer.PROPERTIES_STORAGE_POLICY)) {
properties.put(PropertyAnalyzer.PROPERTIES_STORAGE_POLICY,
olapTable.getStoragePolicy());
}
+ if
(!properties.containsKey(PropertyAnalyzer.PROPERTIES_TDE_ALGORITHM)) {
+ properties.put(PropertyAnalyzer.PROPERTIES_TDE_ALGORITHM,
olapTable.getTDEAlgorithm().name());
+ }
singlePartitionDesc.analyze(partitionInfo.getPartitionColumns().size(),
properties);
partitionInfo.createAndCheckPartitionItem(singlePartitionDesc,
isTempPartition);
@@ -2194,7 +2198,7 @@ public class InternalCatalog implements
CatalogIf<Database> {
tbl.getRowStoreColumnsUniqueIds(rowStoreColumns),
objectPool, tbl.rowStorePageSize(),
tbl.variantEnableFlattenNested(),
- tbl.storagePageSize(),
+ tbl.storagePageSize(), tbl.getTDEAlgorithm(),
tbl.storageDictPageSize());
task.setStorageFormat(tbl.getStorageFormat());
@@ -3136,6 +3140,13 @@ public class InternalCatalog implements
CatalogIf<Database> {
throw new DdlException(e.getMessage());
}
+ try {
+ TEncryptionAlgorithm tdeAlgorithm =
PropertyAnalyzer.analyzeTDEAlgorithm(properties);
+ olapTable.setEncryptionAlgorithm(tdeAlgorithm);
+ } catch (Exception e) {
+ throw new DdlException(e.getMessage());
+ }
+
olapTable.initSchemaColumnUniqueId();
olapTable.initAutoIncrementGenerator(db.getId());
olapTable.rebuildFullSchema();
@@ -4026,6 +4037,13 @@ public class InternalCatalog implements
CatalogIf<Database> {
throw new DdlException(e.getMessage());
}
+ try {
+ TEncryptionAlgorithm tdeAlgorithm =
PropertyAnalyzer.analyzeTDEAlgorithm(properties);
+ olapTable.setEncryptionAlgorithm(tdeAlgorithm);
+ } catch (Exception e) {
+ throw new DdlException(e.getMessage());
+ }
+
olapTable.initSchemaColumnUniqueId();
olapTable.initAutoIncrementGenerator(db.getId());
olapTable.rebuildFullSchema();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java
b/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java
index f290d717446..f91913e5a89 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java
@@ -1081,7 +1081,7 @@ public class ReportHandler extends Daemon {
objectPool,
olapTable.rowStorePageSize(),
olapTable.variantEnableFlattenNested(),
- olapTable.storagePageSize(),
+ olapTable.storagePageSize(),
olapTable.getTDEAlgorithm(),
olapTable.storageDictPageSize());
createReplicaTask.setIsRecoverTask(true);
createReplicaTask.setInvertedIndexFileStorageFormat(olapTable
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
index 9b6f5c545e0..c120271d63a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
@@ -161,6 +161,7 @@ import org.apache.doris.thrift.TDescribeTablesParams;
import org.apache.doris.thrift.TDescribeTablesResult;
import org.apache.doris.thrift.TDropPlsqlPackageRequest;
import org.apache.doris.thrift.TDropPlsqlStoredProcedureRequest;
+import org.apache.doris.thrift.TEncryptionAlgorithm;
import org.apache.doris.thrift.TEncryptionKey;
import org.apache.doris.thrift.TFeResult;
import org.apache.doris.thrift.TFetchResourceResult;
@@ -198,6 +199,8 @@ import org.apache.doris.thrift.TGetMetaTable;
import org.apache.doris.thrift.TGetQueryStatsRequest;
import org.apache.doris.thrift.TGetSnapshotRequest;
import org.apache.doris.thrift.TGetSnapshotResult;
+import org.apache.doris.thrift.TGetTableTDEInfoRequest;
+import org.apache.doris.thrift.TGetTableTDEInfoResult;
import org.apache.doris.thrift.TGetTablesParams;
import org.apache.doris.thrift.TGetTablesResult;
import org.apache.doris.thrift.TGetTabletReplicaInfosRequest;
@@ -4509,4 +4512,46 @@ public class FrontendServiceImpl implements
FrontendService.Iface {
return result;
}
+
+ @Override
+ public TGetTableTDEInfoResult getTableTDEInfo(TGetTableTDEInfoRequest
request) throws TException {
+ String clientAddr = getClientAddrAsString();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("receive getTableTDEInfo request: {}, backend: {}",
request, clientAddr);
+ }
+
+ if (!request.isSetDbId()) {
+ TStatus status = new TStatus()
+ .setStatusCode(TStatusCode.INVALID_ARGUMENT);
+ status.addToErrorMsgs("Missing db id field");
+ return new TGetTableTDEInfoResult().setStatus(status);
+ }
+ if (!request.isSetTableId()) {
+ TStatus status = new TStatus()
+ .setStatusCode(TStatusCode.INVALID_ARGUMENT);
+ status.addToErrorMsgs("Missing db id field");
+ return new TGetTableTDEInfoResult().setStatus(status);
+ }
+ Optional<Database> db =
Env.getCurrentEnv().getInternalCatalog().getDb(request.getDbId());
+ if (!db.isPresent()) {
+ TStatus status = new TStatus()
+ .setStatusCode(TStatusCode.NOT_FOUND);
+ status.addToErrorMsgs("Db=" + request.getDbId() + " not found");
+ return new TGetTableTDEInfoResult().setStatus(status);
+ }
+ Optional<Table> tbl = db.get().getTable(request.getTableId());
+ if (!tbl.isPresent()) {
+ TStatus status = new TStatus()
+ .setStatusCode(TStatusCode.NOT_FOUND);
+ status.addToErrorMsgs("Table=" + request.getTableId() + " not
found");
+ return new TGetTableTDEInfoResult().setStatus(status);
+ }
+
+ TEncryptionAlgorithm tdeAlgorithm = ((OlapTable)
tbl.get()).getTableProperty().getTDEAlgorithm();
+ TStatus status = new TStatus();
+ status.setStatusCode(TStatusCode.OK);
+ TGetTableTDEInfoResult result = new TGetTableTDEInfoResult();
+ result.setAlgorithm(tdeAlgorithm).setStatus(status);
+ return result;
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java
b/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java
index 36e84be5942..96c8a3c18de 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java
@@ -31,6 +31,7 @@ import org.apache.doris.policy.PolicyTypeEnum;
import org.apache.doris.thrift.TColumn;
import org.apache.doris.thrift.TCompressionType;
import org.apache.doris.thrift.TCreateTabletReq;
+import org.apache.doris.thrift.TEncryptionAlgorithm;
import org.apache.doris.thrift.TInvertedIndexFileStorageFormat;
import org.apache.doris.thrift.TInvertedIndexStorageFormat;
import org.apache.doris.thrift.TOlapTableIndex;
@@ -133,6 +134,8 @@ public class CreateReplicaTask extends AgentTask {
private boolean variantEnableFlattenNested;
+ private TEncryptionAlgorithm tdeAlgorithm;
+
public CreateReplicaTask(long backendId, long dbId, long tableId, long
partitionId, long indexId, long tabletId,
long replicaId, short shortKeyColumnCount, int
schemaHash, long version,
KeysType keysType, TStorageType storageType,
@@ -159,7 +162,7 @@ public class CreateReplicaTask extends AgentTask {
Map<Object, Object> objectPool,
long rowStorePageSize,
boolean variantEnableFlattenNested,
- long storagePageSize,
+ long storagePageSize, TEncryptionAlgorithm
tdeAlgorithm,
long storageDictPageSize) {
super(null, backendId, TTaskType.CREATE, dbId, tableId, partitionId,
indexId, tabletId);
@@ -210,6 +213,7 @@ public class CreateReplicaTask extends AgentTask {
this.variantEnableFlattenNested = variantEnableFlattenNested;
this.storagePageSize = storagePageSize;
this.storageDictPageSize = storageDictPageSize;
+ this.tdeAlgorithm = tdeAlgorithm;
}
public void setIsRecoverTask(boolean isRecoverTask) {
@@ -420,6 +424,7 @@ public class CreateReplicaTask extends AgentTask {
createTabletReq.setTimeSeriesCompactionTimeThresholdSeconds(timeSeriesCompactionTimeThresholdSeconds);
createTabletReq.setTimeSeriesCompactionEmptyRowsetsThreshold(timeSeriesCompactionEmptyRowsetsThreshold);
createTabletReq.setTimeSeriesCompactionLevelThreshold(timeSeriesCompactionLevelThreshold);
+ createTabletReq.setTdeAlgorithm(tdeAlgorithm);
if (binlogConfig != null) {
createTabletReq.setBinlogConfig(binlogConfig.toThrift());
diff --git a/fe/fe-core/src/test/java/org/apache/doris/task/AgentTaskTest.java
b/fe/fe-core/src/test/java/org/apache/doris/task/AgentTaskTest.java
index 55c14a658e6..39d822b5449 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/task/AgentTaskTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/task/AgentTaskTest.java
@@ -30,6 +30,7 @@ import org.apache.doris.metric.MetricRepo;
import org.apache.doris.thrift.TAgentTaskRequest;
import org.apache.doris.thrift.TBackend;
import org.apache.doris.thrift.TCompressionType;
+import org.apache.doris.thrift.TEncryptionAlgorithm;
import org.apache.doris.thrift.TStorageMedium;
import org.apache.doris.thrift.TStorageType;
import org.apache.doris.thrift.TTabletType;
@@ -115,7 +116,7 @@ public class AgentTaskTest {
indexId1, tabletId1, replicaId1, shortKeyNum, schemaHash1,
version, KeysType.AGG_KEYS, storageType,
TStorageMedium.SSD, columns, null, 0, latch, null, false,
TTabletType.TABLET_TYPE_DISK, null,
TCompressionType.LZ4F, false, "", false, false, false, "", 0,
0, 0, 0, 0, false, null, null, objectPool, rowStorePageSize, false,
- storagePageSize, storageDictPageSize);
+ storagePageSize, TEncryptionAlgorithm.PLAINTEXT,
storageDictPageSize);
// drop
dropTask = new DropReplicaTask(backendId1, tabletId1, replicaId1,
schemaHash1, false);
diff --git a/gensrc/proto/olap_file.proto b/gensrc/proto/olap_file.proto
index 00d7d5384d8..07d357fdb1d 100644
--- a/gensrc/proto/olap_file.proto
+++ b/gensrc/proto/olap_file.proto
@@ -563,6 +563,7 @@ message TabletMetaPB {
optional int64 time_series_compaction_time_threshold_seconds = 31 [default
= 3600];
optional int64 time_series_compaction_empty_rowsets_threshold = 32
[default = 5];
optional int64 time_series_compaction_level_threshold = 33 [default = 1];
+ optional EncryptionAlgorithmPB encryption_algorithm = 34;
// For cloud
optional int64 index_id = 1000;
@@ -619,6 +620,7 @@ message TabletMetaCloudPB {
reserved 36; // deprecated group_commit_data_bytes
optional int64 time_series_compaction_empty_rowsets_threshold = 37
[default = 5];
optional int64 time_series_compaction_level_threshold = 38 [default = 1];
+ optional EncryptionAlgorithmPB encryption_algorithm = 39;
// Use for selectdb-cloud
optional string table_name = 101;
diff --git a/gensrc/thrift/AgentService.thrift
b/gensrc/thrift/AgentService.thrift
index 67427288a59..ad8cd936132 100644
--- a/gensrc/thrift/AgentService.thrift
+++ b/gensrc/thrift/AgentService.thrift
@@ -222,6 +222,7 @@ struct TCreateTabletReq {
27: optional i64 time_series_compaction_level_threshold = 1
28: optional TInvertedIndexStorageFormat inverted_index_storage_format =
TInvertedIndexStorageFormat.DEFAULT // Deprecated
29: optional Types.TInvertedIndexFileStorageFormat
inverted_index_file_storage_format = Types.TInvertedIndexFileStorageFormat.V2
+ 30: optional TEncryptionAlgorithm tde_algorithm
// For cloud
1000: optional bool is_in_memory = false
diff --git a/gensrc/thrift/FrontendService.thrift
b/gensrc/thrift/FrontendService.thrift
index 9c9676d72b0..ebad46253f7 100644
--- a/gensrc/thrift/FrontendService.thrift
+++ b/gensrc/thrift/FrontendService.thrift
@@ -1746,4 +1746,6 @@ service FrontendService {
TFetchRoutineLoadJobResult fetchRoutineLoadJob(1:
TFetchRoutineLoadJobRequest request)
TGetEncryptionKeysResult getEncryptionKeys(1: TGetEncryptionKeysRequest
request)
+
+ TGetTableTDEInfoResult getTableTDEInfo(1: TGetTableTDEInfoRequest request)
}
diff --git a/regression-test/data/query_p0/system/test_table_properties.out
b/regression-test/data/query_p0/system/test_table_properties.out
index b4981be72a8..598a4ea0ea5 100644
Binary files a/regression-test/data/query_p0/system/test_table_properties.out
and b/regression-test/data/query_p0/system/test_table_properties.out differ
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
