This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new b4beec8ea86 [fix](OrcWriter) fix be core when upgrading BE without
upgrading FE (#40303)
b4beec8ea86 is described below
commit b4beec8ea867a381e37fe50a003470179380d040
Author: Tiewei Fang <43782773+bepppo...@users.noreply.github.com>
AuthorDate: Wed Sep 4 10:24:41 2024 +0800
[fix](OrcWriter) fix be core when upgrading BE without upgrading FE (#40303)
bp: #40282
---
be/src/vec/sink/vresult_sink.h | 4 ++++
be/src/vec/sink/writer/vfile_result_writer.cpp | 5 +++++
.../src/main/java/org/apache/doris/analysis/OutFileClause.java | 1 +
gensrc/thrift/DataSinks.thrift | 7 +++++++
4 files changed, 17 insertions(+)
diff --git a/be/src/vec/sink/vresult_sink.h b/be/src/vec/sink/vresult_sink.h
index a3563bbb501..8c28e848a85 100644
--- a/be/src/vec/sink/vresult_sink.h
+++ b/be/src/vec/sink/vresult_sink.h
@@ -73,6 +73,7 @@ struct ResultFileOptions {
std::string file_suffix;
//Bring BOM when exporting to CSV format
bool with_bom = false;
+ int64_t orc_writer_version = 0;
ResultFileOptions(const TResultFileSinkOptions& t_opt) {
file_path = t_opt.file_path;
@@ -123,6 +124,9 @@ struct ResultFileOptions {
if (t_opt.__isset.orc_compression_type) {
orc_compression_type = t_opt.orc_compression_type;
}
+ if (t_opt.__isset.orc_writer_version) {
+ orc_writer_version = t_opt.orc_writer_version;
+ }
}
};
diff --git a/be/src/vec/sink/writer/vfile_result_writer.cpp
b/be/src/vec/sink/writer/vfile_result_writer.cpp
index acaa05963c1..0daaa27cc09 100644
--- a/be/src/vec/sink/writer/vfile_result_writer.cpp
+++ b/be/src/vec/sink/writer/vfile_result_writer.cpp
@@ -81,6 +81,11 @@ VFileResultWriter::VFileResultWriter(const
ResultFileOptions* file_opts,
Status VFileResultWriter::open(RuntimeState* state, RuntimeProfile* profile) {
_state = state;
_init_profile(profile);
+ // check orc writer version
+ if (_file_opts->file_format == TFileFormatType::FORMAT_ORC &&
+ _file_opts->orc_writer_version < 1) {
+ return Status::InternalError("orc writer version is less than 1.");
+ }
// Delete existing files
if (_file_opts->delete_existing_files) {
RETURN_IF_ERROR(_delete_dir());
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java
index 837610dfdf0..2038b266a7f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java
@@ -878,6 +878,7 @@ public class OutFileClause {
if (isOrcFormat()) {
sinkOptions.setOrcSchema(serializeOrcSchema());
sinkOptions.setOrcCompressionType(orcCompressionType);
+ sinkOptions.setOrcWriterVersion(1);
}
return sinkOptions;
}
diff --git a/gensrc/thrift/DataSinks.thrift b/gensrc/thrift/DataSinks.thrift
index 25be947ba34..96c079b7738 100644
--- a/gensrc/thrift/DataSinks.thrift
+++ b/gensrc/thrift/DataSinks.thrift
@@ -133,6 +133,13 @@ struct TResultFileSinkOptions {
18: optional bool with_bom;
19: optional PlanNodes.TFileCompressType orc_compression_type;
+
+ // Since we have changed the type mapping from Doris to Orc type,
+ // using the Outfile to export Date/Datetime types will cause BE core dump
+ // when only upgrading BE without upgrading FE.
+ // orc_writer_version = 1 means doris FE is higher than version 2.1.5
+ // orc_writer_version = 0 means doris FE is less than or equal to version
2.1.5
+ 20: optional i64 orc_writer_version;
}
struct TMemoryScratchSink {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
