This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 2543b569bbc [Optimize](Row store) pick #37145, #38236 (#38932)
2543b569bbc is described below
commit 2543b569bbc26ef235cd985c8fd27b05b24dfa5d
Author: Xr Ling <63634816+lxr...@users.noreply.github.com>
AuthorDate: Wed Aug 7 09:55:42 2024 +0800
[Optimize](Row store) pick #37145, #38236 (#38932)
---
be/src/common/config.cpp | 2 -
be/src/common/config.h | 2 -
be/src/olap/rowset/segment_v2/options.h | 2 +
be/src/olap/rowset/segment_v2/segment_writer.cpp | 5 +-
.../rowset/segment_v2/vertical_segment_writer.cpp | 5 +-
be/src/olap/tablet_meta.cpp | 3 +
be/src/olap/tablet_schema.cpp | 4 ++
be/src/olap/tablet_schema.h | 4 ++
.../java/org/apache/doris/alter/RollupJobV2.java | 3 +-
.../org/apache/doris/alter/SchemaChangeJobV2.java | 3 +-
.../java/org/apache/doris/backup/RestoreJob.java | 3 +-
.../main/java/org/apache/doris/catalog/Env.java | 4 ++
.../java/org/apache/doris/catalog/OlapTable.java | 15 +++++
.../org/apache/doris/catalog/TableProperty.java | 14 +++++
.../apache/doris/common/util/PropertyAnalyzer.java | 29 ++++++++++
.../apache/doris/datasource/InternalCatalog.java | 16 +++++-
.../org/apache/doris/master/ReportHandler.java | 3 +-
.../org/apache/doris/task/CreateReplicaTask.java | 7 ++-
.../java/org/apache/doris/task/AgentTaskTest.java | 3 +-
gensrc/proto/olap_file.proto | 1 +
gensrc/thrift/AgentService.thrift | 1 +
.../query_p0/show/test_row_store_page_size.groovy | 67 ++++++++++++++++++++++
22 files changed, 183 insertions(+), 13 deletions(-)
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index 6420a624127..e1b44043505 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -1053,8 +1053,6 @@ DEFINE_mInt64(max_tablet_io_errors, "-1");
DEFINE_Int32(tablet_path_check_interval_seconds, "-1");
DEFINE_mInt32(tablet_path_check_batch_size, "1000");
-// Page size of row column, default 4KB
-DEFINE_mInt64(row_column_page_size, "4096");
// it must be larger than or equal to 5MB
DEFINE_mInt32(s3_write_buffer_size, "5242880");
// The timeout config for S3 buffer allocation
diff --git a/be/src/common/config.h b/be/src/common/config.h
index d1f91ab693d..d226623f0e5 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -1097,8 +1097,6 @@ DECLARE_mInt64(max_tablet_io_errors);
DECLARE_Int32(tablet_path_check_interval_seconds);
DECLARE_mInt32(tablet_path_check_batch_size);
-// Page size of row column, default 4KB
-DECLARE_mInt64(row_column_page_size);
// it must be larger than or equal to 5MB
DECLARE_mInt32(s3_write_buffer_size);
// The timeout config for S3 buffer allocation
diff --git a/be/src/olap/rowset/segment_v2/options.h
b/be/src/olap/rowset/segment_v2/options.h
index 19041f4c51d..e68e4f6b6bc 100644
--- a/be/src/olap/rowset/segment_v2/options.h
+++ b/be/src/olap/rowset/segment_v2/options.h
@@ -24,6 +24,8 @@ namespace segment_v2 {
static constexpr size_t DEFAULT_PAGE_SIZE = 1024 * 1024; // default size: 1M
+constexpr long ROW_STORE_PAGE_SIZE_DEFAULT_VALUE = 16384; // default row
column page size: 16KB
+
struct PageBuilderOptions {
size_t data_page_size = DEFAULT_PAGE_SIZE;
diff --git a/be/src/olap/rowset/segment_v2/segment_writer.cpp
b/be/src/olap/rowset/segment_v2/segment_writer.cpp
index 38b79f47f10..76a072644be 100644
--- a/be/src/olap/rowset/segment_v2/segment_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/segment_writer.cpp
@@ -256,8 +256,11 @@ Status SegmentWriter::init(const std::vector<uint32_t>&
col_ids, bool has_key) {
if (column.is_row_store_column()) {
// smaller page size for row store column
- opts.data_page_size = config::row_column_page_size;
+ auto page_size = _tablet_schema->row_store_page_size();
+ opts.data_page_size =
+ (page_size > 0) ? page_size :
segment_v2::ROW_STORE_PAGE_SIZE_DEFAULT_VALUE;
}
+
std::unique_ptr<ColumnWriter> writer;
RETURN_IF_ERROR(ColumnWriter::create(opts, &column, _file_writer,
&writer));
RETURN_IF_ERROR(writer->init());
diff --git a/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp
b/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp
index a6f9ceee7c5..6bc4445b626 100644
--- a/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp
@@ -213,8 +213,11 @@ Status
VerticalSegmentWriter::_create_column_writer(uint32_t cid, const TabletCo
if (column.is_row_store_column()) {
// smaller page size for row store column
- opts.data_page_size = config::row_column_page_size;
+ auto page_size = _tablet_schema->row_store_page_size();
+ opts.data_page_size =
+ (page_size > 0) ? page_size :
segment_v2::ROW_STORE_PAGE_SIZE_DEFAULT_VALUE;
}
+
std::unique_ptr<ColumnWriter> writer;
RETURN_IF_ERROR(ColumnWriter::create(opts, &column, _file_writer,
&writer));
RETURN_IF_ERROR(writer->init());
diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp
index 72bf0f0ee39..4f20cbc01f5 100644
--- a/be/src/olap/tablet_meta.cpp
+++ b/be/src/olap/tablet_meta.cpp
@@ -298,6 +298,9 @@ TabletMeta::TabletMeta(int64_t table_id, int64_t
partition_id, int64_t tablet_id
if (tablet_schema.__isset.store_row_column) {
schema->set_store_row_column(tablet_schema.store_row_column);
}
+ if (tablet_schema.__isset.row_store_page_size) {
+ schema->set_row_store_page_size(tablet_schema.row_store_page_size);
+ }
if (tablet_schema.__isset.skip_write_index_on_load) {
schema->set_skip_write_index_on_load(tablet_schema.skip_write_index_on_load);
}
diff --git a/be/src/olap/tablet_schema.cpp b/be/src/olap/tablet_schema.cpp
index a2ef2649b4d..62d80fb28c1 100644
--- a/be/src/olap/tablet_schema.cpp
+++ b/be/src/olap/tablet_schema.cpp
@@ -993,6 +993,7 @@ void TabletSchema::init_from_pb(const TabletSchemaPB&
schema, bool ignore_extrac
_sort_type = schema.sort_type();
_sort_col_num = schema.sort_col_num();
_compression_type = schema.compression_type();
+ _row_store_page_size = schema.row_store_page_size();
_schema_version = schema.schema_version();
// Default to V1 inverted index storage format for backward compatibility
if not specified in schema.
if (!schema.has_inverted_index_storage_format()) {
@@ -1051,6 +1052,7 @@ void TabletSchema::build_current_tablet_schema(int64_t
index_id, int32_t version
_skip_write_index_on_load = ori_tablet_schema.skip_write_index_on_load();
_sort_type = ori_tablet_schema.sort_type();
_sort_col_num = ori_tablet_schema.sort_col_num();
+ _row_store_page_size = ori_tablet_schema.row_store_page_size();
// copy from table_schema_param
_schema_version = version;
@@ -1204,6 +1206,7 @@ void TabletSchema::to_schema_pb(TabletSchemaPB*
tablet_schema_pb) const {
tablet_schema_pb->set_sort_col_num(_sort_col_num);
tablet_schema_pb->set_schema_version(_schema_version);
tablet_schema_pb->set_compression_type(_compression_type);
+ tablet_schema_pb->set_row_store_page_size(_row_store_page_size);
tablet_schema_pb->set_version_col_idx(_version_col_idx);
tablet_schema_pb->set_inverted_index_storage_format(_inverted_index_storage_format);
}
@@ -1505,6 +1508,7 @@ bool operator==(const TabletSchema& a, const
TabletSchema& b) {
if (a._disable_auto_compaction != b._disable_auto_compaction) return false;
if (a._enable_single_replica_compaction !=
b._enable_single_replica_compaction) return false;
if (a._store_row_column != b._store_row_column) return false;
+ if (a._row_store_page_size != b._row_store_page_size) return false;
if (a._skip_write_index_on_load != b._skip_write_index_on_load) return
false;
return true;
}
diff --git a/be/src/olap/tablet_schema.h b/be/src/olap/tablet_schema.h
index bd3b1f6ca4e..ed7ab896107 100644
--- a/be/src/olap/tablet_schema.h
+++ b/be/src/olap/tablet_schema.h
@@ -36,6 +36,7 @@
#include "common/status.h"
#include "gutil/stringprintf.h"
#include "olap/olap_common.h"
+#include "olap/rowset/segment_v2/options.h"
#include "runtime/define_primitive_type.h"
#include "runtime/descriptors.h"
#include "util/string_util.h"
@@ -338,6 +339,8 @@ public:
void set_version_col_idx(int32_t version_col_idx) { _version_col_idx =
version_col_idx; }
int32_t version_col_idx() const { return _version_col_idx; }
segment_v2::CompressionTypePB compression_type() const { return
_compression_type; }
+ void set_row_store_page_size(long page_size) { _row_store_page_size =
page_size; }
+ long row_store_page_size() const { return _row_store_page_size; }
const std::vector<TabletIndex>& indexes() const { return _indexes; }
bool has_inverted_index() const {
@@ -482,6 +485,7 @@ private:
size_t _num_rows_per_row_block = 0;
CompressKind _compress_kind = COMPRESS_NONE;
segment_v2::CompressionTypePB _compression_type =
segment_v2::CompressionTypePB::LZ4F;
+ long _row_store_page_size = segment_v2::ROW_STORE_PAGE_SIZE_DEFAULT_VALUE;
size_t _next_column_unique_id = 0;
std::string _auto_increment_column;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java
b/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java
index 2ffd5e5a4ec..3081c401bc8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java
@@ -278,7 +278,8 @@ public class RollupJobV2 extends AlterJobV2 implements
GsonPostProcessable {
tbl.getTimeSeriesCompactionEmptyRowsetsThreshold(),
tbl.getTimeSeriesCompactionLevelThreshold(),
tbl.storeRowColumn(),
- binlogConfig, objectPool);
+ binlogConfig, objectPool,
+ tbl.rowStorePageSize());
createReplicaTask.setBaseTablet(tabletIdMap.get(rollupTabletId),
baseSchemaHash);
if (this.storageFormat != null) {
createReplicaTask.setStorageFormat(this.storageFormat);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
index 919fc673648..86f0e24a627 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
@@ -283,7 +283,8 @@ public class SchemaChangeJobV2 extends AlterJobV2 {
tbl.getTimeSeriesCompactionEmptyRowsetsThreshold(),
tbl.getTimeSeriesCompactionLevelThreshold(),
tbl.storeRowColumn(),
- binlogConfig, objectPool);
+ binlogConfig, objectPool,
+ tbl.rowStorePageSize());
createReplicaTask.setBaseTablet(partitionIndexTabletMap.get(partitionId,
shadowIdxId)
.get(shadowTabletId), originSchemaHash);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
index 57696bc5a84..e4ff8225a75 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
@@ -1102,7 +1102,8 @@ public class RestoreJob extends AbstractJob {
localTbl.getTimeSeriesCompactionEmptyRowsetsThreshold(),
localTbl.getTimeSeriesCompactionLevelThreshold(),
localTbl.storeRowColumn(),
- binlogConfig, objectPool);
+ binlogConfig, objectPool,
+ localTbl.rowStorePageSize());
task.setInvertedIndexStorageFormat(localTbl.getInvertedIndexStorageFormat());
task.setInRestoreMode(true);
batchTask.addTask(task);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
index dae7e732497..f828b6ae576 100755
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
@@ -3384,6 +3384,10 @@ public class Env {
if (olapTable.storeRowColumn()) {
sb.append(",\n\"").append(PropertyAnalyzer.PROPERTIES_STORE_ROW_COLUMN).append("\"
= \"");
sb.append(olapTable.storeRowColumn()).append("\"");
+
+ // row store page size
+
sb.append(",\n\"").append(PropertyAnalyzer.PROPERTIES_ROW_STORE_PAGE_SIZE).append("\"
= \"");
+ sb.append(olapTable.rowStorePageSize()).append("\"");
}
// skip inverted index on load
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
index cfc57a98991..92a9059e611 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
@@ -2432,6 +2432,21 @@ public class OlapTable extends Table implements
MTMVRelatedTableIf {
tableProperty.buildCompressionType();
}
+ public void setRowStorePageSize(long pageSize) {
+ TableProperty tableProperty = getOrCreatTableProperty();
+
tableProperty.modifyTableProperties(PropertyAnalyzer.PROPERTIES_ROW_STORE_PAGE_SIZE,
+ Long.valueOf(pageSize).toString());
+ tableProperty.buildRowStorePageSize();
+ }
+
+ public long rowStorePageSize() {
+ if (tableProperty != null) {
+ return tableProperty.rowStorePageSize();
+ }
+ return PropertyAnalyzer.ROW_STORE_PAGE_SIZE_DEFAULT_VALUE;
+ }
+
+
public void setStorageFormat(TStorageFormat storageFormat) {
TableProperty tableProperty = getOrCreatTableProperty();
tableProperty.modifyTableProperties(PropertyAnalyzer.PROPERTIES_STORAGE_FORMAT,
storageFormat.name());
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
index 510fd7903ae..2ede77a6180 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
@@ -92,6 +92,8 @@ public class TableProperty implements Writable {
private boolean skipWriteIndexOnLoad = false;
+ private long rowStorePageSize =
PropertyAnalyzer.ROW_STORE_PAGE_SIZE_DEFAULT_VALUE;
+
private String compactionPolicy =
PropertyAnalyzer.SIZE_BASED_COMPACTION_POLICY;
private long timeSeriesCompactionGoalSizeMbytes
@@ -238,6 +240,17 @@ public class TableProperty implements Writable {
return storeRowColumn;
}
+ public TableProperty buildRowStorePageSize() {
+ rowStorePageSize = Long.parseLong(
+
properties.getOrDefault(PropertyAnalyzer.PROPERTIES_ROW_STORE_PAGE_SIZE,
+
Long.toString(PropertyAnalyzer.ROW_STORE_PAGE_SIZE_DEFAULT_VALUE)));
+ return this;
+ }
+
+ public long rowStorePageSize() {
+ return rowStorePageSize;
+ }
+
public TableProperty buildSkipWriteIndexOnLoad() {
skipWriteIndexOnLoad = Boolean.parseBoolean(
properties.getOrDefault(PropertyAnalyzer.PROPERTIES_SKIP_WRITE_INDEX_ON_LOAD,
"false"));
@@ -606,6 +619,7 @@ public class TableProperty implements Writable {
.buildBinlogConfig()
.buildEnableLightSchemaChange()
.buildStoreRowColumn()
+ .buildRowStorePageSize()
.buildSkipWriteIndexOnLoad()
.buildCompactionPolicy()
.buildTimeSeriesCompactionGoalSizeMbytes()
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
index 93efad3293d..d1a710bcf9f 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
@@ -90,6 +90,10 @@ public class PropertyAnalyzer {
public static final String PROPERTIES_TIMEOUT = "timeout";
public static final String PROPERTIES_COMPRESSION = "compression";
+ // row store page size, default 16KB
+ public static final String PROPERTIES_ROW_STORE_PAGE_SIZE =
"row_store_page_size";
+ public static final long ROW_STORE_PAGE_SIZE_DEFAULT_VALUE = 16384;
+
public static final String PROPERTIES_ENABLE_LIGHT_SCHEMA_CHANGE =
"light_schema_change";
public static final String PROPERTIES_DISTRIBUTION_TYPE =
"distribution_type";
@@ -887,6 +891,31 @@ public class PropertyAnalyzer {
}
}
+ public static long alignTo4K(long size) {
+ return (size + 4095) & ~4095;
+ }
+
+ // analyzeRowStorePageSize will parse the row_store_page_size from
properties
+ public static long analyzeRowStorePageSize(Map<String, String> properties)
throws AnalysisException {
+ long rowStorePageSize = ROW_STORE_PAGE_SIZE_DEFAULT_VALUE;
+ if (properties != null &&
properties.containsKey(PROPERTIES_ROW_STORE_PAGE_SIZE)) {
+ String rowStorePageSizeStr =
properties.get(PROPERTIES_ROW_STORE_PAGE_SIZE);
+ try {
+ rowStorePageSize =
alignTo4K(Long.parseLong(rowStorePageSizeStr));
+ } catch (NumberFormatException e) {
+ throw new AnalysisException("Invalid row store page size: " +
rowStorePageSizeStr);
+ }
+
+ if (rowStorePageSize <= 0) {
+ throw new AnalysisException("Row store page size should larger
than 0.");
+ }
+
+ properties.remove(PROPERTIES_ROW_STORE_PAGE_SIZE);
+ }
+
+ return rowStorePageSize;
+ }
+
// analyzeStorageFormat will parse the storage format from properties
// sql: alter table tablet_name set ("storage_format" = "v2")
// Use this sql to convert all tablets(base and rollup index) to a new
format segment
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
index 992a8232dfb..5a5ae14ba5f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
@@ -1511,6 +1511,10 @@ public class InternalCatalog implements
CatalogIf<Database> {
properties.put(PropertyAnalyzer.PROPERTIES_STORE_ROW_COLUMN,
olapTable.storeRowColumn().toString());
}
+ if
(!properties.containsKey(PropertyAnalyzer.PROPERTIES_ROW_STORE_PAGE_SIZE)) {
+ properties.put(PropertyAnalyzer.PROPERTIES_ROW_STORE_PAGE_SIZE,
+ Long.toString(olapTable.rowStorePageSize()));
+ }
if
(!properties.containsKey(PropertyAnalyzer.PROPERTIES_SKIP_WRITE_INDEX_ON_LOAD))
{
properties.put(PropertyAnalyzer.PROPERTIES_SKIP_WRITE_INDEX_ON_LOAD,
olapTable.skipWriteIndexOnLoad().toString());
@@ -1988,7 +1992,7 @@ public class InternalCatalog implements
CatalogIf<Database> {
tbl.getTimeSeriesCompactionTimeThresholdSeconds(),
tbl.getTimeSeriesCompactionEmptyRowsetsThreshold(),
tbl.getTimeSeriesCompactionLevelThreshold(),
- tbl.storeRowColumn(), binlogConfig, objectPool);
+ tbl.storeRowColumn(), binlogConfig, objectPool,
tbl.rowStorePageSize());
task.setStorageFormat(tbl.getStorageFormat());
task.setInvertedIndexStorageFormat(tbl.getInvertedIndexStorageFormat());
@@ -2347,6 +2351,16 @@ public class InternalCatalog implements
CatalogIf<Database> {
}
olapTable.setCompressionType(compressionType);
+ // get row_store_page_size
+ long rowStorePageSize =
PropertyAnalyzer.ROW_STORE_PAGE_SIZE_DEFAULT_VALUE;
+ try {
+ rowStorePageSize =
PropertyAnalyzer.analyzeRowStorePageSize(properties);
+ } catch (AnalysisException e) {
+ throw new DdlException(e.getMessage());
+ }
+
+ olapTable.setRowStorePageSize(rowStorePageSize);
+
// check data sort properties
int keyColumnSize =
CollectionUtils.isEmpty(keysDesc.getClusterKeysColumnIds()) ?
keysDesc.keysColumnSize() :
keysDesc.getClusterKeysColumnIds().size();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java
b/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java
index de56b4ce9b6..29887658f7a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java
@@ -865,7 +865,8 @@ public class ReportHandler extends Daemon {
olapTable.getTimeSeriesCompactionEmptyRowsetsThreshold(),
olapTable.getTimeSeriesCompactionLevelThreshold(),
olapTable.storeRowColumn(),
- binlogConfig, objectPool);
+ binlogConfig, objectPool,
+ olapTable.rowStorePageSize());
createReplicaTask.setIsRecoverTask(true);
createReplicaTask.setInvertedIndexStorageFormat(olapTable
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java
b/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java
index 1de5d4e8d7d..e6a3cb8b090 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java
@@ -64,6 +64,7 @@ public class CreateReplicaTask extends AgentTask {
private TStorageType storageType;
private TStorageMedium storageMedium;
private TCompressionType compressionType;
+ private long rowStorePageSize;
private List<Column> columns;
@@ -148,7 +149,8 @@ public class CreateReplicaTask extends AgentTask {
long timeSeriesCompactionLevelThreshold,
boolean storeRowColumn,
BinlogConfig binlogConfig,
- Map<Object, Object> objectPool) {
+ Map<Object, Object> objectPool,
+ long rowStorePageSize) {
super(null, backendId, TTaskType.CREATE, dbId, tableId, partitionId,
indexId, tabletId);
this.replicaId = replicaId;
@@ -193,6 +195,8 @@ public class CreateReplicaTask extends AgentTask {
this.storeRowColumn = storeRowColumn;
this.binlogConfig = binlogConfig;
this.objectPool = objectPool;
+ this.rowStorePageSize = rowStorePageSize;
+
}
public void setIsRecoverTask(boolean isRecoverTask) {
@@ -334,6 +338,7 @@ public class CreateReplicaTask extends AgentTask {
tSchema.setEnableSingleReplicaCompaction(enableSingleReplicaCompaction);
tSchema.setSkipWriteIndexOnLoad(skipWriteIndexOnLoad);
tSchema.setStoreRowColumn(storeRowColumn);
+ tSchema.setRowStorePageSize(rowStorePageSize);
createTabletReq.setTabletSchema(tSchema);
createTabletReq.setVersion(version);
diff --git a/fe/fe-core/src/test/java/org/apache/doris/task/AgentTaskTest.java
b/fe/fe-core/src/test/java/org/apache/doris/task/AgentTaskTest.java
index b604076ddba..ad5da36334e 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/task/AgentTaskTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/task/AgentTaskTest.java
@@ -73,6 +73,7 @@ public class AgentTaskTest {
private long version = 1L;
private TStorageType storageType = TStorageType.COLUMN;
+ private long rowStorePageSize = 16384L;
private List<Column> columns;
private MarkedCountDownLatch<Long, Long> latch = new
MarkedCountDownLatch<Long, Long>(3);
@@ -107,7 +108,7 @@ public class AgentTaskTest {
createReplicaTask = new CreateReplicaTask(backendId1, dbId, tableId,
partitionId,
indexId1, tabletId1, replicaId1, shortKeyNum, schemaHash1,
version, KeysType.AGG_KEYS, storageType,
TStorageMedium.SSD, columns, null, 0, latch, null, false,
TTabletType.TABLET_TYPE_DISK, null,
- TCompressionType.LZ4F, false, "", false, false, false, "", 0,
0, 0, 0, 0, false, null, objectPool);
+ TCompressionType.LZ4F, false, "", false, false, false, "", 0,
0, 0, 0, 0, false, null, objectPool, rowStorePageSize);
// drop
dropTask = new DropReplicaTask(backendId1, tabletId1, replicaId1,
schemaHash1, false);
diff --git a/gensrc/proto/olap_file.proto b/gensrc/proto/olap_file.proto
index 327a79c0bca..4d0d3707132 100644
--- a/gensrc/proto/olap_file.proto
+++ b/gensrc/proto/olap_file.proto
@@ -268,6 +268,7 @@ message TabletSchemaPB {
optional InvertedIndexStorageFormatPB inverted_index_storage_format = 25
[default=V1];
// column unique ids for row store columns
repeated int32 row_store_column_unique_ids = 26;
+ optional int64 row_store_page_size = 27 [default=16384];
}
enum TabletStatePB {
diff --git a/gensrc/thrift/AgentService.thrift
b/gensrc/thrift/AgentService.thrift
index 6da55c66c5a..8c9a943d9ac 100644
--- a/gensrc/thrift/AgentService.thrift
+++ b/gensrc/thrift/AgentService.thrift
@@ -45,6 +45,7 @@ struct TTabletSchema {
17: optional bool enable_single_replica_compaction = false
18: optional bool skip_write_index_on_load = false
19: optional list<i32> cluster_key_idxes
+ 21: optional i64 row_store_page_size = 16384
}
// this enum stands for different storage format in src_backends
diff --git
a/regression-test/suites/query_p0/show/test_row_store_page_size.groovy
b/regression-test/suites/query_p0/show/test_row_store_page_size.groovy
new file mode 100644
index 00000000000..909f68d0357
--- /dev/null
+++ b/regression-test/suites/query_p0/show/test_row_store_page_size.groovy
@@ -0,0 +1,67 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.codehaus.groovy.runtime.IOGroovyMethods
+
+suite ("test_row_store_page_size") {
+
+ sql """ DROP TABLE IF EXISTS ps_table_1; """
+
+ sql """
+ create table ps_table_1(
+ k1 int not null,
+ k2 int not null,
+ k3 bigint null,
+ k4 varchar(100) null
+ )
+ unique key (k1,k2)
+ distributed BY hash(k1) buckets 3
+ properties("replication_num" = "1", "store_row_column" = "true");
+ """
+
+ test {
+ sql "show create table ps_table_1;"
+ check { result, exception, startTime, endTime ->
+ assertTrue(result[0][1].contains("\"row_store_page_size\" =
\"16384\""))
+ }
+ }
+
+ sql """ DROP TABLE IF EXISTS ps_table_1; """
+
+ sql """ DROP TABLE IF EXISTS ps_table_2; """
+
+ sql """
+ create table ps_table_2(
+ k1 int not null,
+ k2 int not null,
+ k3 bigint null,
+ k4 varchar(100) null
+ )
+ unique key (k1,k2)
+ distributed BY hash(k1) buckets 3
+ properties("replication_num" = "1", "store_row_column" = "true",
"row_store_page_size" = "8190");
+ """
+
+ test {
+ sql "show create table ps_table_2;"
+ check { result, exception, startTime, endTime ->
+ assertTrue(result[0][1].contains("\"row_store_page_size\" =
\"8192\""))
+ }
+ }
+
+ sql """ DROP TABLE IF EXISTS ps_table_2; """
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
