This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch compaction_opt
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/compaction_opt by this push:
new 86c724a930 [bugfix](compaction) fix add rowset type check (#14267)
86c724a930 is described below
commit 86c724a930f066308bfca458598eb0cd50b05275
Author: yixiutt <102007456+yixi...@users.noreply.github.com>
AuthorDate: Tue Nov 15 14:18:19 2022 +0800
[bugfix](compaction) fix add rowset type check (#14267)
---
be/src/olap/rowset/beta_rowset_writer.cpp | 1 +
be/src/olap/rowset/rowset_factory.cpp | 13 ++++++-------
be/src/olap/rowset/rowset_factory.h | 2 +-
be/src/olap/snapshot_manager.cpp | 2 +-
be/src/olap/tablet.cpp | 7 +++----
be/test/olap/rowid_conversion_test.cpp | 4 ++--
be/test/olap/rowset/beta_rowset_test.cpp | 2 +-
be/test/olap/segcompaction_test.cpp | 4 ++--
be/test/vec/olap/vertical_compaction_test.cpp | 12 ++++++------
build-support/clang-format.sh | 3 +--
gensrc/proto/olap_file.proto | 1 -
11 files changed, 24 insertions(+), 27 deletions(-)
diff --git a/be/src/olap/rowset/beta_rowset_writer.cpp
b/be/src/olap/rowset/beta_rowset_writer.cpp
index 154d76224d..55e80007ff 100644
--- a/be/src/olap/rowset/beta_rowset_writer.cpp
+++ b/be/src/olap/rowset/beta_rowset_writer.cpp
@@ -972,6 +972,7 @@ Status
BetaRowsetWriter::_flush_segment_writer(std::unique_ptr<segment_v2::Segme
DCHECK_LE(min_key.compare(max_key), 0);
key_bounds.set_min_key(min_key.to_string());
key_bounds.set_max_key(max_key.to_string());
+ _segments_encoded_key_bounds.emplace_back(key_bounds);
Statistics segstat;
segstat.row_num = row_num;
diff --git a/be/src/olap/rowset/rowset_factory.cpp
b/be/src/olap/rowset/rowset_factory.cpp
index 5df3acf4f9..fcbfb5fd61 100644
--- a/be/src/olap/rowset/rowset_factory.cpp
+++ b/be/src/olap/rowset/rowset_factory.cpp
@@ -32,27 +32,26 @@ Status RowsetFactory::create_rowset(TabletSchemaSPtr
schema, const std::string&
if (rowset_meta->rowset_type() == ALPHA_ROWSET) {
return Status::OLAPInternalError(OLAP_ERR_ROWSET_INVALID);
}
- if (rowset_meta->rowset_type() == BETA_ROWSET ||
- rowset_meta->rowset_type() == VERTICAL_BETA_ROWSET) {
+ if (rowset_meta->rowset_type() == BETA_ROWSET) {
rowset->reset(new BetaRowset(schema, tablet_path, rowset_meta));
return (*rowset)->init();
}
return Status::OLAPInternalError(OLAP_ERR_ROWSET_TYPE_NOT_FOUND); //
should never happen
}
-Status RowsetFactory::create_rowset_writer(const RowsetWriterContext& context,
+Status RowsetFactory::create_rowset_writer(const RowsetWriterContext& context,
bool is_vertical,
std::unique_ptr<RowsetWriter>*
output) {
if (context.rowset_type == ALPHA_ROWSET) {
return Status::OLAPInternalError(OLAP_ERR_ROWSET_INVALID);
}
if (context.rowset_type == BETA_ROWSET) {
+ if (is_vertical) {
+ output->reset(new VerticalBetaRowsetWriter);
+ return (*output)->init(context);
+ }
output->reset(new BetaRowsetWriter);
return (*output)->init(context);
}
- if (context.rowset_type == VERTICAL_BETA_ROWSET) {
- output->reset(new VerticalBetaRowsetWriter);
- return (*output)->init(context);
- }
return Status::OLAPInternalError(OLAP_ERR_ROWSET_TYPE_NOT_FOUND);
}
diff --git a/be/src/olap/rowset/rowset_factory.h
b/be/src/olap/rowset/rowset_factory.h
index e216b81f52..72d2e76cf8 100644
--- a/be/src/olap/rowset/rowset_factory.h
+++ b/be/src/olap/rowset/rowset_factory.h
@@ -37,7 +37,7 @@ public:
// create and init rowset writer.
// return OLAP_SUCCESS and set `*output` to inited rowset writer.
// return others if failed
- static Status create_rowset_writer(const RowsetWriterContext& context,
+ static Status create_rowset_writer(const RowsetWriterContext& context,
bool is_vertical,
std::unique_ptr<RowsetWriter>* output);
};
diff --git a/be/src/olap/snapshot_manager.cpp b/be/src/olap/snapshot_manager.cpp
index 904ef62baf..66a740feab 100644
--- a/be/src/olap/snapshot_manager.cpp
+++ b/be/src/olap/snapshot_manager.cpp
@@ -266,7 +266,7 @@ Status SnapshotManager::_rename_rowset_id(const
RowsetMetaPB& rs_meta_pb,
context.segments_overlap = rowset_meta->segments_overlap();
std::unique_ptr<RowsetWriter> rs_writer;
- RETURN_NOT_OK(RowsetFactory::create_rowset_writer(context, &rs_writer));
+ RETURN_NOT_OK(RowsetFactory::create_rowset_writer(context, false,
&rs_writer));
res = rs_writer->add_rowset(org_rowset);
if (!res.ok()) {
diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp
index 14275c9e95..d741203f1c 100644
--- a/be/src/olap/tablet.cpp
+++ b/be/src/olap/tablet.cpp
@@ -1610,8 +1610,7 @@ Status Tablet::create_vertical_rowset_writer(
context.tablet_schema = tablet_schema;
context.enable_unique_key_merge_on_write =
enable_unique_key_merge_on_write();
_init_context_common_fields(context);
- context.rowset_type = VERTICAL_BETA_ROWSET;
- return RowsetFactory::create_rowset_writer(context, rowset_writer);
+ return RowsetFactory::create_rowset_writer(context, true, rowset_writer);
}
Status Tablet::create_rowset_writer(const Version& version, const
RowsetStatePB& rowset_state,
@@ -1639,7 +1638,7 @@ Status Tablet::create_rowset_writer(const Version&
version, const RowsetStatePB&
context.enable_unique_key_merge_on_write =
enable_unique_key_merge_on_write();
context.fs = fs;
_init_context_common_fields(context);
- return RowsetFactory::create_rowset_writer(context, rowset_writer);
+ return RowsetFactory::create_rowset_writer(context, false, rowset_writer);
}
Status Tablet::create_rowset_writer(const int64_t& txn_id, const PUniqueId&
load_id,
@@ -1657,7 +1656,7 @@ Status Tablet::create_rowset_writer(const int64_t&
txn_id, const PUniqueId& load
context.tablet_schema = tablet_schema;
context.enable_unique_key_merge_on_write =
enable_unique_key_merge_on_write();
_init_context_common_fields(context);
- return RowsetFactory::create_rowset_writer(context, rowset_writer);
+ return RowsetFactory::create_rowset_writer(context, false, rowset_writer);
}
void Tablet::_init_context_common_fields(RowsetWriterContext& context) {
diff --git a/be/test/olap/rowid_conversion_test.cpp
b/be/test/olap/rowid_conversion_test.cpp
index 01ac4a667c..a08f17f4c4 100644
--- a/be/test/olap/rowid_conversion_test.cpp
+++ b/be/test/olap/rowid_conversion_test.cpp
@@ -164,7 +164,7 @@ protected:
create_rowset_writer_context(tablet_schema, overlap, UINT32_MAX,
&writer_context);
std::unique_ptr<RowsetWriter> rowset_writer;
- Status s = RowsetFactory::create_rowset_writer(writer_context,
&rowset_writer);
+ Status s = RowsetFactory::create_rowset_writer(writer_context, false,
&rowset_writer);
EXPECT_TRUE(s.ok());
RowCursor input_row;
@@ -367,7 +367,7 @@ protected:
RowsetWriterContext writer_context;
create_rowset_writer_context(tablet_schema, NONOVERLAPPING, 3456,
&writer_context);
std::unique_ptr<RowsetWriter> output_rs_writer;
- Status s = RowsetFactory::create_rowset_writer(writer_context,
&output_rs_writer);
+ Status s = RowsetFactory::create_rowset_writer(writer_context, false,
&output_rs_writer);
EXPECT_TRUE(s.ok());
// merge input rowset
diff --git a/be/test/olap/rowset/beta_rowset_test.cpp
b/be/test/olap/rowset/beta_rowset_test.cpp
index 2df689fc04..ea8a3485fa 100644
--- a/be/test/olap/rowset/beta_rowset_test.cpp
+++ b/be/test/olap/rowset/beta_rowset_test.cpp
@@ -182,7 +182,7 @@ TEST_F(BetaRowsetTest, BasicFunctionTest) {
create_rowset_writer_context(tablet_schema, &writer_context);
std::unique_ptr<RowsetWriter> rowset_writer;
- s = RowsetFactory::create_rowset_writer(writer_context,
&rowset_writer);
+ s = RowsetFactory::create_rowset_writer(writer_context, false,
&rowset_writer);
EXPECT_EQ(Status::OK(), s);
RowCursor input_row;
diff --git a/be/test/olap/segcompaction_test.cpp
b/be/test/olap/segcompaction_test.cpp
index ca2dc28f04..fa07a514ec 100644
--- a/be/test/olap/segcompaction_test.cpp
+++ b/be/test/olap/segcompaction_test.cpp
@@ -216,7 +216,7 @@ TEST_F(SegCompactionTest, SegCompactionThenRead) {
create_rowset_writer_context(10047, tablet_schema, &writer_context);
std::unique_ptr<RowsetWriter> rowset_writer;
- s = RowsetFactory::create_rowset_writer(writer_context,
&rowset_writer);
+ s = RowsetFactory::create_rowset_writer(writer_context, false,
&rowset_writer);
EXPECT_EQ(Status::OK(), s);
RowCursor input_row;
@@ -322,7 +322,7 @@ TEST_F(SegCompactionTest,
SegCompactionInterleaveWithBig_ooooOOoOooooooooO) {
create_rowset_writer_context(10048, tablet_schema, &writer_context);
std::unique_ptr<RowsetWriter> rowset_writer;
- s = RowsetFactory::create_rowset_writer(writer_context,
&rowset_writer);
+ s = RowsetFactory::create_rowset_writer(writer_context, false,
&rowset_writer);
EXPECT_EQ(Status::OK(), s);
RowCursor input_row;
diff --git a/be/test/vec/olap/vertical_compaction_test.cpp
b/be/test/vec/olap/vertical_compaction_test.cpp
index 7061b6f471..28e605ef8f 100644
--- a/be/test/vec/olap/vertical_compaction_test.cpp
+++ b/be/test/vec/olap/vertical_compaction_test.cpp
@@ -161,7 +161,7 @@ protected:
RowsetId rowset_id;
rowset_id.init(inc_id);
rowset_writer_context->rowset_id = rowset_id;
- rowset_writer_context->rowset_type = VERTICAL_BETA_ROWSET;
+ rowset_writer_context->rowset_type = BETA_ROWSET;
rowset_writer_context->data_dir = _data_dir.get();
rowset_writer_context->rowset_state = VISIBLE;
rowset_writer_context->tablet_schema = tablet_schema;
@@ -198,7 +198,7 @@ protected:
create_rowset_writer_context(tablet_schema, overlap, UINT32_MAX,
&writer_context);
std::unique_ptr<RowsetWriter> rowset_writer;
- Status s = RowsetFactory::create_rowset_writer(writer_context,
&rowset_writer);
+ Status s = RowsetFactory::create_rowset_writer(writer_context, true,
&rowset_writer);
EXPECT_TRUE(s.ok());
RowCursor input_row;
@@ -493,7 +493,7 @@ TEST_F(VerticalCompactionTest, TestDupKeyVerticalMerge) {
RowsetWriterContext writer_context;
create_rowset_writer_context(tablet_schema, NONOVERLAPPING, 3456,
&writer_context);
std::unique_ptr<RowsetWriter> output_rs_writer;
- Status s = RowsetFactory::create_rowset_writer(writer_context,
&output_rs_writer);
+ Status s = RowsetFactory::create_rowset_writer(writer_context, true,
&output_rs_writer);
EXPECT_TRUE(s.ok());
// merge input rowset
@@ -601,7 +601,7 @@ TEST_F(VerticalCompactionTest, TestUniqueKeyVerticalMerge) {
RowsetWriterContext writer_context;
create_rowset_writer_context(tablet_schema, NONOVERLAPPING, 3456,
&writer_context);
std::unique_ptr<RowsetWriter> output_rs_writer;
- Status s = RowsetFactory::create_rowset_writer(writer_context,
&output_rs_writer);
+ Status s = RowsetFactory::create_rowset_writer(writer_context, true,
&output_rs_writer);
EXPECT_TRUE(s.ok());
// merge input rowset
@@ -702,7 +702,7 @@ TEST_F(VerticalCompactionTest,
TestDupKeyVerticalMergeWithDelete) {
RowsetWriterContext writer_context;
create_rowset_writer_context(tablet_schema, NONOVERLAPPING, 3456,
&writer_context);
std::unique_ptr<RowsetWriter> output_rs_writer;
- Status s = RowsetFactory::create_rowset_writer(writer_context,
&output_rs_writer);
+ Status s = RowsetFactory::create_rowset_writer(writer_context, true,
&output_rs_writer);
EXPECT_TRUE(s.ok());
// merge input rowset
@@ -797,7 +797,7 @@ TEST_F(VerticalCompactionTest, TestAggKeyVerticalMerge) {
RowsetWriterContext writer_context;
create_rowset_writer_context(tablet_schema, NONOVERLAPPING, 3456,
&writer_context);
std::unique_ptr<RowsetWriter> output_rs_writer;
- Status s = RowsetFactory::create_rowset_writer(writer_context,
&output_rs_writer);
+ Status s = RowsetFactory::create_rowset_writer(writer_context, true,
&output_rs_writer);
EXPECT_TRUE(s.ok());
// merge input rowset
diff --git a/build-support/clang-format.sh b/build-support/clang-format.sh
index dd04d4ba23..9dfeb25167 100755
--- a/build-support/clang-format.sh
+++ b/build-support/clang-format.sh
@@ -31,7 +31,6 @@ DORIS_HOME=$(
)
export DORIS_HOME
-#CLANG_FORMAT="${CLANG_FORMAT_BINARY:=$(which clang-format)}"
-CLANG_FORMAT="/mnt/disk1/yixiu/incubator-doris/clang-format"
+CLANG_FORMAT="${CLANG_FORMAT_BINARY:=$(which clang-format)}"
python "${DORIS_HOME}/build-support/run_clang_format.py"
"--clang-format-executable" "${CLANG_FORMAT}" "-r" "--style" "file" "--inplace"
"true" "--extensions" "c,h,C,H,cpp,hpp,cc,hh,c++,h++,cxx,hxx" "--exclude"
"none" "be/src be/test"
diff --git a/gensrc/proto/olap_file.proto b/gensrc/proto/olap_file.proto
index 46715ffa96..30b127fc78 100644
--- a/gensrc/proto/olap_file.proto
+++ b/gensrc/proto/olap_file.proto
@@ -34,7 +34,6 @@ message ZoneMap {
enum RowsetTypePB {
ALPHA_ROWSET = 0; // doris original column storage format
BETA_ROWSET = 1; // new column storage format
- VERTICAL_BETA_ROWSET = 2; // vertical for compaction
}
enum RowsetStatePB {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
