This is an automated email from the ASF dual-hosted git repository.
zhangchen pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new c7acf31b95e [cherry-pick](branch-2.0) fix data correctness risk when
load delete sign data into a table with sequence col (#32574) (#36228)
c7acf31b95e is described below
commit c7acf31b95ecf46aa39fa1bc6e3f16bfafe384bf
Author: zhannngchen <48427519+zhannngc...@users.noreply.github.com>
AuthorDate: Thu Jun 13 14:21:24 2024 +0800
[cherry-pick](branch-2.0) fix data correctness risk when load delete sign
data into a table with sequence col (#32574) (#36228)
cherry-pick #32574
---
be/src/olap/rowset/segment_v2/segment_writer.cpp | 12 +++++++-----
.../test_partial_update_seq_col_delete.out | Bin 1064 -> 1526 bytes
.../test_partial_update_seq_type_delete.out | Bin 3017 -> 2967 bytes
.../test_partial_update_seq_col_delete.groovy | 8 ++++++++
4 files changed, 15 insertions(+), 5 deletions(-)
diff --git a/be/src/olap/rowset/segment_v2/segment_writer.cpp
b/be/src/olap/rowset/segment_v2/segment_writer.cpp
index b696a88602d..2f3fb67ae62 100644
--- a/be/src/olap/rowset/segment_v2/segment_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/segment_writer.cpp
@@ -484,10 +484,12 @@ Status
SegmentWriter::append_block_with_partial_content(const vectorized::Block*
return st;
}
- // if the delete sign is marked, it means that the value columns of
the row
- // will not be read. So we don't need to read the missing values from
the previous rows.
- // But we still need to mark the previous row on delete bitmap
- if (have_delete_sign) {
+ // 1. if the delete sign is marked, it means that the value columns of
the row will not
+ // be read. So we don't need to read the missing values from the
previous rows.
+ // 2. the one exception is when there are sequence columns in the
table, we need to read
+ // the sequence columns, otherwise it may cause the merge-on-read
based compaction
+ // policy to produce incorrect results
+ if (have_delete_sign && !_tablet_schema->has_sequence_col()) {
has_default_or_nullable = true;
use_default_or_null_flag.emplace_back(true);
} else {
@@ -666,7 +668,7 @@ Status
SegmentWriter::fill_missing_columns(vectorized::MutableColumns& mutable_f
(delete_sign_column_data != nullptr &&
delete_sign_column_data[read_index[idx + segment_start_pos]] !=
0)) {
for (auto i = 0; i < cids_missing.size(); ++i) {
- // if the column has default value, fiil it with default value
+ // if the column has default value, fill it with default value
// otherwise, if the column is nullable, fill it with null
value
const auto& tablet_column =
_tablet_schema->column(cids_missing[i]);
if (tablet_column.has_default_value()) {
diff --git
a/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_seq_col_delete.out
b/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_seq_col_delete.out
index a59dfbf5203..fe5746c41b7 100644
Binary files
a/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_seq_col_delete.out
and
b/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_seq_col_delete.out
differ
diff --git
a/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_seq_type_delete.out
b/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_seq_type_delete.out
index 80f580661fc..05f0c5dab4e 100644
Binary files
a/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_seq_type_delete.out
and
b/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_seq_type_delete.out
differ
diff --git
a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_seq_col_delete.groovy
b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_seq_col_delete.groovy
index c6cfdba1106..dca646948b4 100644
---
a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_seq_col_delete.groovy
+++
b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_seq_col_delete.groovy
@@ -80,6 +80,14 @@ suite("test_primary_key_partial_update_seq_col_delete",
"p0") {
select * from ${tableName} order by id;
"""
+ sql "SET show_hidden_columns=true"
+
+ sql "sync"
+ qt_partial_update_without_seq_hidden_columns """
+ select * from ${tableName} order by id;
+ """
+
+ sql "SET show_hidden_columns=false"
// provide the sequence column this time, should update according
to the
// given sequence values
streamLoad {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
