This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new e08de52ee7 [chore](compile) using PCH for compilation acceleration
under clang (#19303)
e08de52ee7 is described below
commit e08de52ee7631c8f3fd498a05c63f817ec94aaaf
Author: DeadlineFen <117912096+deadline...@users.noreply.github.com>
AuthorDate: Mon May 8 19:51:06 2023 +0800
[chore](compile) using PCH for compilation acceleration under clang (#19303)
---
.licenserc.yaml | 1 +
be/CMakeLists.txt | 26 +-
be/src/agent/CMakeLists.txt | 2 +
be/src/common/CMakeLists.txt | 2 +
be/src/exec/CMakeLists.txt | 2 +
be/src/exprs/CMakeLists.txt | 2 +
be/src/gen_cpp/CMakeLists.txt | 2 +-
be/src/geo/CMakeLists.txt | 8 +-
be/src/gutil/atomicops.h | 9 +-
be/src/http/CMakeLists.txt | 2 +
be/src/io/CMakeLists.txt | 2 +
be/src/olap/CMakeLists.txt | 2 +
be/src/olap/base_compaction.cpp | 8 +-
be/src/olap/compaction.cpp | 26 +-
be/src/olap/cumulative_compaction.cpp | 6 +-
be/src/olap/delta_writer.cpp | 18 +-
be/src/olap/memtable.cpp | 8 +-
be/src/olap/merger.cpp | 22 +-
be/src/olap/olap_define.h | 29 --
be/src/olap/options.cpp | 11 +-
be/src/olap/push_handler.cpp | 2 +-
be/src/olap/reader.cpp | 2 +-
be/src/olap/row_cursor.cpp | 14 +-
be/src/olap/rowset/CMakeLists.txt | 5 +-
be/src/olap/rowset/beta_rowset_reader.cpp | 8 +-
be/src/olap/rowset/beta_rowset_writer.cpp | 34 +-
be/src/olap/rowset/rowset.cpp | 4 +-
be/src/olap/rowset/segcompaction.cpp | 20 +-
be/src/olap/schema_change.cpp | 8 +-
be/src/olap/segment_loader.cpp | 2 +-
be/src/olap/segment_loader.h | 2 +-
be/src/olap/snapshot_manager.cpp | 18 +-
be/src/olap/storage_engine.cpp | 4 +-
be/src/olap/tablet.cpp | 18 +-
be/src/olap/tablet_manager.cpp | 31 +-
be/src/olap/tablet_meta.cpp | 2 +-
be/src/olap/task/engine_checksum_task.cpp | 4 +-
be/src/olap/txn_manager.cpp | 4 +-
be/src/olap/types.h | 2 +-
be/src/pch/pch.cc | 0
be/src/pch/pch.h | 527 +++++++++++++++++++++
be/src/pipeline/CMakeLists.txt | 4 +-
be/src/runtime/CMakeLists.txt | 4 +-
be/src/service/CMakeLists.txt | 4 +
be/src/tools/CMakeLists.txt | 2 +
be/src/util/CMakeLists.txt | 7 +-
be/src/util/string_parser.cpp | 48 --
be/src/util/string_parser.hpp | 31 +-
be/src/util/thrift_util.cpp | 6 +
be/src/util/trace_metrics.cpp | 3 -
be/src/vec/CMakeLists.txt | 4 +-
be/src/vec/olap/block_reader.cpp | 4 +-
be/src/vec/olap/vcollect_iterator.cpp | 8 +-
be/src/vec/olap/vertical_block_reader.cpp | 4 +-
build.sh | 3 +
.../install/source-install/compilation-general.md | 17 +
.../install/source-install/compilation-general.md | 17 +
env.sh | 6 +
run-be-ut.sh | 3 +
59 files changed, 825 insertions(+), 249 deletions(-)
diff --git a/.licenserc.yaml b/.licenserc.yaml
index 130e58bcac..d52378d5b5 100644
--- a/.licenserc.yaml
+++ b/.licenserc.yaml
@@ -67,6 +67,7 @@ header:
- "be/src/util/sse2neon.h"
- "be/src/util/utf8_check.cpp"
- "be/src/util/cityhash102"
+ - "be/src/pch/*"
- "build-support/run_clang_format.py"
- "regression-test/data"
- "docs/.vuepress/public/css/animate.min.css"
diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt
index 90d487b1cb..f781f99c76 100644
--- a/be/CMakeLists.txt
+++ b/be/CMakeLists.txt
@@ -652,11 +652,11 @@ include_directories(
${THIRDPARTY_DIR}/include
${GPERFTOOLS_HOME}/include
)
-include_directories($ENV{JAVA_HOME}/include)
+include_directories(${DORIS_JAVA_HOME}/include)
if (NOT OS_MACOSX)
- include_directories($ENV{JAVA_HOME}/include/linux)
+ include_directories(${DORIS_JAVA_HOME}/include/linux)
else()
- include_directories($ENV{JAVA_HOME}/include/darwin)
+ include_directories(${DORIS_JAVA_HOME}/include/darwin)
endif()
if (NOT OS_MACOSX)
@@ -976,6 +976,26 @@ else()
message(STATUS "Link Flags: ${TEST_LINK_LIBS}")
endif()
+if (ENABLE_PCH)
+add_library(pch STATIC ${SRC_DIR}pch/pch.cc)
+ target_precompile_headers(
+ pch
+ PUBLIC
+ ${SRC_DIR}pch/pch.h
+ )
+ if (COMPILER_CLANG)
+ add_compile_options(-Xclang -fno-pch-timestamp)
+ else()
+ add_compile_options(-fpch-preprocess)
+ endif()
+endif()
+
+function(pch_reuse target)
+ if (ENABLE_PCH)
+ target_precompile_headers(${target} REUSE_FROM pch)
+ endif()
+endfunction(pch_reuse target)
+
add_subdirectory(${SRC_DIR}/agent)
add_subdirectory(${SRC_DIR}/common)
add_subdirectory(${SRC_DIR}/exec)
diff --git a/be/src/agent/CMakeLists.txt b/be/src/agent/CMakeLists.txt
index 60d8ace8b1..538b1d4d0a 100644
--- a/be/src/agent/CMakeLists.txt
+++ b/be/src/agent/CMakeLists.txt
@@ -35,3 +35,5 @@ if (OS_MACOSX)
endif()
add_library(Agent STATIC ${AGENT_SOURCES})
+
+pch_reuse(Agent)
\ No newline at end of file
diff --git a/be/src/common/CMakeLists.txt b/be/src/common/CMakeLists.txt
index 1d2254c689..8c4513eef1 100644
--- a/be/src/common/CMakeLists.txt
+++ b/be/src/common/CMakeLists.txt
@@ -27,5 +27,7 @@ add_library(Common STATIC
exception.cpp
)
+pch_reuse(Common)
+
# Generate env_config.h according to env_config.h.in
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/env_config.h.in
${GENSRC_DIR}/common/env_config.h)
diff --git a/be/src/exec/CMakeLists.txt b/be/src/exec/CMakeLists.txt
index 2ce6e15697..6f4a383748 100644
--- a/be/src/exec/CMakeLists.txt
+++ b/be/src/exec/CMakeLists.txt
@@ -67,3 +67,5 @@ endif()
add_library(Exec STATIC
${EXEC_FILES}
)
+
+pch_reuse(Exec)
\ No newline at end of file
diff --git a/be/src/exprs/CMakeLists.txt b/be/src/exprs/CMakeLists.txt
index 7903d07048..bda3ba9df0 100644
--- a/be/src/exprs/CMakeLists.txt
+++ b/be/src/exprs/CMakeLists.txt
@@ -30,3 +30,5 @@ add_library(Exprs
string_functions.cpp
json_functions.cpp
)
+
+pch_reuse(Exprs)
\ No newline at end of file
diff --git a/be/src/gen_cpp/CMakeLists.txt b/be/src/gen_cpp/CMakeLists.txt
index 0b4d42da5a..437377f146 100644
--- a/be/src/gen_cpp/CMakeLists.txt
+++ b/be/src/gen_cpp/CMakeLists.txt
@@ -32,7 +32,7 @@ add_library(DorisGen STATIC ${SRC_FILES})
# Setting these files as code-generated lets make clean and incremental builds
work
# correctly
-set_source_files_properties(${SRC_FILES} PROPERTIES GENERATED TRUE)
+# set_source_files_properties(${SRC_FILES} PROPERTIES GENERATED TRUE)
#add_dependencies(DorisGen thrift-cpp)
#add_dependencies(Opcode function)
diff --git a/be/src/geo/CMakeLists.txt b/be/src/geo/CMakeLists.txt
index 978c634329..fbe5d52f9a 100644
--- a/be/src/geo/CMakeLists.txt
+++ b/be/src/geo/CMakeLists.txt
@@ -21,17 +21,21 @@ set(LIBRARY_OUTPUT_PATH "${BUILD_DIR}/src/geo")
# where to put generated binaries
set(EXECUTABLE_OUTPUT_PATH "${BUILD_DIR}/src/geo")
+add_library(geo_type STATIC geo_types.cpp)
+
add_library(Geo STATIC
geo_common.cpp
- geo_types.cpp
wkt_parse.cpp
wkb_parse.cpp
${GENSRC_DIR}/geo/wkt_lex.l.cpp
${GENSRC_DIR}/geo/wkt_yacc.y.cpp
geo_tobinary.cpp
ByteOrderValues.cpp
- machine.h)
+ machine.h
+)
+pch_reuse(Geo)
+target_link_libraries(Geo geo_type)
include(CheckCXXCompilerFlag)
set(WARNING_OPTION "-Wno-unused-but-set-variable")
check_cxx_compiler_flag(${WARNING_OPTION} HAS_WARNING_OPTION)
diff --git a/be/src/gutil/atomicops.h b/be/src/gutil/atomicops.h
index 4c6d93e28c..5e7c606792 100644
--- a/be/src/gutil/atomicops.h
+++ b/be/src/gutil/atomicops.h
@@ -66,18 +66,19 @@
// #endif
// ------------------------------------------------------------------------
-#define GCC_VERSION (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 +
__GNUC_PATCHLEVEL__)
+#define GUTILS_GCC_VERSION (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 +
__GNUC_PATCHLEVEL__)
-#define CLANG_VERSION (__clang_major__ * 10000 + __clang_minor__ * 100 +
__clang_patchlevel__)
+#define GUTILS_CLANG_VERSION \
+ (__clang_major__ * 10000 + __clang_minor__ * 100 + __clang_patchlevel__)
// ThreadSanitizer provides own implementation of atomicops.
#if defined(THREAD_SANITIZER)
#include "gutil/atomicops-internals-tsan.h" // IWYU pragma: export
#elif defined(__GNUC__) && (defined(__i386) || defined(__x86_64__))
#include "gutil/atomicops-internals-x86.h" // IWYU pragma: export
-#elif defined(__GNUC__) && GCC_VERSION >= 40700
+#elif defined(__GNUC__) && GUTILS_GCC_VERSION >= 40700
#include "gutil/atomicops-internals-gcc.h" // IWYU pragma: export
-#elif defined(__clang__) && CLANG_VERSION >= 30400
+#elif defined(__clang__) && GUTILS_CLANG_VERSION >= 30400
#include "gutil/atomicops-internals-gcc.h" // IWYU pragma: export
#else
#error You need to implement atomic operations for this architecture
diff --git a/be/src/http/CMakeLists.txt b/be/src/http/CMakeLists.txt
index e563f1dadf..0ae56e86e9 100644
--- a/be/src/http/CMakeLists.txt
+++ b/be/src/http/CMakeLists.txt
@@ -57,3 +57,5 @@ add_library(Webserver STATIC
action/version_action.cpp
action/jeprofile_actions.cpp
action/file_cache_action.cpp)
+
+pch_reuse(Webserver)
\ No newline at end of file
diff --git a/be/src/io/CMakeLists.txt b/be/src/io/CMakeLists.txt
index 6d784844d0..45f66901c4 100644
--- a/be/src/io/CMakeLists.txt
+++ b/be/src/io/CMakeLists.txt
@@ -61,3 +61,5 @@ set(IO_FILES
add_library(IO STATIC
${IO_FILES}
)
+
+pch_reuse(IO)
\ No newline at end of file
diff --git a/be/src/olap/CMakeLists.txt b/be/src/olap/CMakeLists.txt
index 91ae9293c2..6398692be2 100644
--- a/be/src/olap/CMakeLists.txt
+++ b/be/src/olap/CMakeLists.txt
@@ -114,3 +114,5 @@ add_library(Olap STATIC
if (NOT USE_MEM_TRACKER)
target_compile_options(Olap PRIVATE -Wno-unused-lambda-capture)
endif()
+
+pch_reuse(Olap)
\ No newline at end of file
diff --git a/be/src/olap/base_compaction.cpp b/be/src/olap/base_compaction.cpp
index c1fcbe3f33..883ede79a1 100644
--- a/be/src/olap/base_compaction.cpp
+++ b/be/src/olap/base_compaction.cpp
@@ -55,7 +55,7 @@ Status BaseCompaction::prepare_compact() {
TRACE("got base compaction lock");
// 1. pick rowsets to compact
- RETURN_NOT_OK(pick_rowsets_to_compact());
+ RETURN_IF_ERROR(pick_rowsets_to_compact());
TRACE("rowsets picked");
TRACE_COUNTER_INCREMENT("input_rowsets_count", _input_rowsets.size());
_tablet->set_clone_occurred(false);
@@ -87,7 +87,7 @@ Status BaseCompaction::execute_compact_impl() {
// 2. do base compaction, merge rowsets
int64_t permits = get_compaction_permits();
- RETURN_NOT_OK(do_compaction(permits));
+ RETURN_IF_ERROR(do_compaction(permits));
TRACE("compaction finished");
// 3. set state to success
@@ -126,8 +126,8 @@ void BaseCompaction::_filter_input_rowset() {
Status BaseCompaction::pick_rowsets_to_compact() {
_input_rowsets = _tablet->pick_candidate_rowsets_to_base_compaction();
- RETURN_NOT_OK(check_version_continuity(_input_rowsets));
- RETURN_NOT_OK(_check_rowset_overlapping(_input_rowsets));
+ RETURN_IF_ERROR(check_version_continuity(_input_rowsets));
+ RETURN_IF_ERROR(_check_rowset_overlapping(_input_rowsets));
_filter_input_rowset();
if (_input_rowsets.size() <= 1) {
return Status::Error<BE_NO_SUITABLE_VERSION>();
diff --git a/be/src/olap/compaction.cpp b/be/src/olap/compaction.cpp
index af556a5392..4dfda3bc16 100644
--- a/be/src/olap/compaction.cpp
+++ b/be/src/olap/compaction.cpp
@@ -72,8 +72,8 @@ Compaction::Compaction(const TabletSharedPtr& tablet, const
std::string& label)
Compaction::~Compaction() {}
Status Compaction::compact() {
- RETURN_NOT_OK(prepare_compact());
- RETURN_NOT_OK(execute_compact());
+ RETURN_IF_ERROR(prepare_compact());
+ RETURN_IF_ERROR(execute_compact());
return Status::OK();
}
@@ -165,7 +165,7 @@ bool Compaction::is_rowset_tidy(std::string& pre_max_key,
const RowsetSharedPtr&
Status Compaction::do_compact_ordered_rowsets() {
build_basic_info();
RowsetWriterContext ctx;
- RETURN_NOT_OK(construct_output_rowset_writer(ctx));
+ RETURN_IF_ERROR(construct_output_rowset_writer(ctx));
LOG(INFO) << "start to do ordered data compaction, tablet=" <<
_tablet->full_name()
<< ", output_version=" << _output_version;
@@ -173,8 +173,8 @@ Status Compaction::do_compact_ordered_rowsets() {
auto seg_id = 0;
std::vector<KeyBoundsPB> segment_key_bounds;
for (auto rowset : _input_rowsets) {
- RETURN_NOT_OK(rowset->link_files_to(_tablet->tablet_path(),
_output_rs_writer->rowset_id(),
- seg_id));
+ RETURN_IF_ERROR(rowset->link_files_to(_tablet->tablet_path(),
+ _output_rs_writer->rowset_id(),
seg_id));
seg_id += rowset->num_segments();
std::vector<KeyBoundsPB> key_bounds;
@@ -269,7 +269,7 @@ Status Compaction::do_compaction_impl(int64_t permits) {
OlapStopWatch watch;
if (handle_ordered_data_compaction()) {
- RETURN_NOT_OK(modify_rowsets());
+ RETURN_IF_ERROR(modify_rowsets());
TRACE("modify rowsets finished");
int64_t now = UnixMillis();
@@ -295,8 +295,8 @@ Status Compaction::do_compaction_impl(int64_t permits) {
<< ", output_version=" << _output_version << ", permits: " <<
permits;
bool vertical_compaction = should_vertical_compaction();
RowsetWriterContext ctx;
- RETURN_NOT_OK(construct_input_rowset_readers());
- RETURN_NOT_OK(construct_output_rowset_writer(ctx, vertical_compaction));
+ RETURN_IF_ERROR(construct_input_rowset_readers());
+ RETURN_IF_ERROR(construct_output_rowset_writer(ctx, vertical_compaction));
if (compaction_type() == ReaderType::READER_COLD_DATA_COMPACTION) {
Tablet::add_pending_remote_rowset(_output_rs_writer->rowset_id().to_string());
}
@@ -345,7 +345,7 @@ Status Compaction::do_compaction_impl(int64_t permits) {
TRACE("output rowset built");
// 3. check correctness
- RETURN_NOT_OK(check_correctness(stats));
+ RETURN_IF_ERROR(check_correctness(stats));
TRACE("check correctness finished");
if (_input_row_num > 0 && stats.rowid_conversion &&
config::inverted_index_compaction_enable) {
@@ -413,7 +413,7 @@ Status Compaction::do_compaction_impl(int64_t permits) {
}
// 4. modify rowsets in memory
- RETURN_NOT_OK(modify_rowsets(&stats));
+ RETURN_IF_ERROR(modify_rowsets(&stats));
TRACE("modify rowsets finished");
// 5. update last success compaction time
@@ -494,7 +494,7 @@ Status
Compaction::construct_output_rowset_writer(RowsetWriterContext& ctx, bool
Status Compaction::construct_input_rowset_readers() {
for (auto& rowset : _input_rowsets) {
RowsetReaderSharedPtr rs_reader;
- RETURN_NOT_OK(rowset->create_reader(&rs_reader));
+ RETURN_IF_ERROR(rowset->create_reader(&rs_reader));
_input_rs_readers.push_back(std::move(rs_reader));
}
return Status::OK();
@@ -552,11 +552,11 @@ Status Compaction::modify_rowsets(const
Merger::Statistics* stats) {
RETURN_IF_ERROR(_tablet->check_rowid_conversion(_output_rowset,
location_map));
_tablet->merge_delete_bitmap(output_rowset_delete_bitmap);
- RETURN_NOT_OK(_tablet->modify_rowsets(output_rowsets,
_input_rowsets, true));
+ RETURN_IF_ERROR(_tablet->modify_rowsets(output_rowsets,
_input_rowsets, true));
}
} else {
std::lock_guard<std::shared_mutex> wrlock(_tablet->get_header_lock());
- RETURN_NOT_OK(_tablet->modify_rowsets(output_rowsets, _input_rowsets,
true));
+ RETURN_IF_ERROR(_tablet->modify_rowsets(output_rowsets,
_input_rowsets, true));
}
{
diff --git a/be/src/olap/cumulative_compaction.cpp
b/be/src/olap/cumulative_compaction.cpp
index 303b58eefe..ecc5ebb7eb 100644
--- a/be/src/olap/cumulative_compaction.cpp
+++ b/be/src/olap/cumulative_compaction.cpp
@@ -61,7 +61,7 @@ Status CumulativeCompaction::prepare_compact() {
<< _tablet->cumulative_layer_point() << ", tablet=" <<
_tablet->full_name();
// 2. pick rowsets to compact
- RETURN_NOT_OK(pick_rowsets_to_compact());
+ RETURN_IF_ERROR(pick_rowsets_to_compact());
TRACE("rowsets picked");
TRACE_COUNTER_INCREMENT("input_rowsets_count", _input_rowsets.size());
_tablet->set_clone_occurred(false);
@@ -88,7 +88,7 @@ Status CumulativeCompaction::execute_compact_impl() {
// 3. do cumulative compaction, merge rowsets
int64_t permits = get_compaction_permits();
- RETURN_NOT_OK(do_compaction(permits));
+ RETURN_IF_ERROR(do_compaction(permits));
TRACE("compaction finished");
// 4. set state to success
@@ -117,7 +117,7 @@ Status CumulativeCompaction::pick_rowsets_to_compact() {
// candidate_rowsets may not be continuous
// So we need to choose the longest continuous path from it.
std::vector<Version> missing_versions;
- RETURN_NOT_OK(find_longest_consecutive_version(&candidate_rowsets,
&missing_versions));
+ RETURN_IF_ERROR(find_longest_consecutive_version(&candidate_rowsets,
&missing_versions));
if (!missing_versions.empty()) {
DCHECK(missing_versions.size() == 2);
LOG(WARNING) << "There are missed versions among rowsets. "
diff --git a/be/src/olap/delta_writer.cpp b/be/src/olap/delta_writer.cpp
index 1454da843b..4fb5100078 100644
--- a/be/src/olap/delta_writer.cpp
+++ b/be/src/olap/delta_writer.cpp
@@ -158,8 +158,8 @@ Status DeltaWriter::init() {
return Status::Error<TRY_LOCK_FAILED>();
}
std::lock_guard<std::mutex> push_lock(_tablet->get_push_lock());
-
RETURN_NOT_OK(_storage_engine->txn_manager()->prepare_txn(_req.partition_id,
_tablet,
- _req.txn_id,
_req.load_id));
+
RETURN_IF_ERROR(_storage_engine->txn_manager()->prepare_txn(_req.partition_id,
_tablet,
+
_req.txn_id, _req.load_id));
}
if (_tablet->enable_unique_key_merge_on_write() && _delete_bitmap ==
nullptr) {
_delete_bitmap.reset(new DeleteBitmap(_tablet->tablet_id()));
@@ -178,7 +178,7 @@ Status DeltaWriter::init() {
context.is_direct_write = true;
context.mow_context =
std::make_shared<MowContext>(_cur_max_version, _rowset_ids,
_delete_bitmap);
- RETURN_NOT_OK(_tablet->create_rowset_writer(context, &_rowset_writer));
+ RETURN_IF_ERROR(_tablet->create_rowset_writer(context, &_rowset_writer));
_schema.reset(new Schema(_tablet_schema));
_reset_mem_table();
@@ -187,7 +187,7 @@ Status DeltaWriter::init() {
// unique key should flush serial because we need to make sure same key
should sort
// in the same order in all replica.
bool should_serial = _tablet->keys_type() == KeysType::UNIQUE_KEYS;
-
RETURN_NOT_OK(_storage_engine->memtable_flush_executor()->create_flush_token(
+
RETURN_IF_ERROR(_storage_engine->memtable_flush_executor()->create_flush_token(
&_flush_token, _rowset_writer->type(), should_serial,
_req.is_high_priority));
_is_init = true;
@@ -205,7 +205,7 @@ Status DeltaWriter::write(const vectorized::Block* block,
const std::vector<int>
}
std::lock_guard<std::mutex> l(_lock);
if (!_is_init && !_is_cancelled) {
- RETURN_NOT_OK(init());
+ RETURN_IF_ERROR(init());
}
if (_is_cancelled) {
@@ -269,7 +269,7 @@ Status DeltaWriter::flush_memtable_and_wait(bool need_wait)
{
if (need_wait) {
// wait all memtables in flush queue to be flushed.
- RETURN_NOT_OK(_flush_token->wait());
+ RETURN_IF_ERROR(_flush_token->wait());
}
return Status::OK();
}
@@ -286,7 +286,7 @@ Status DeltaWriter::wait_flush() {
return _cancel_status;
}
}
- RETURN_NOT_OK(_flush_token->wait());
+ RETURN_IF_ERROR(_flush_token->wait());
return Status::OK();
}
@@ -327,7 +327,7 @@ Status DeltaWriter::close() {
// in same partition has data loaded.
// so we have to also init this DeltaWriter, so that it can create an
empty rowset
// for this tablet when being closed.
- RETURN_NOT_OK(init());
+ RETURN_IF_ERROR(init());
}
if (_is_cancelled) {
@@ -343,7 +343,7 @@ Status DeltaWriter::close() {
auto s = _flush_memtable_async();
_mem_table.reset();
_is_closed = true;
- if (OLAP_UNLIKELY(!s.ok())) {
+ if (UNLIKELY(!s.ok())) {
return s;
} else {
return Status::OK();
diff --git a/be/src/olap/memtable.cpp b/be/src/olap/memtable.cpp
index 85ac69b16c..b09fe144f6 100644
--- a/be/src/olap/memtable.cpp
+++ b/be/src/olap/memtable.cpp
@@ -409,11 +409,11 @@ Status MemTable::flush() {
// and new segment ids is between [atomic_num_segments_before_flush,
atomic_num_segments_after_flush),
// and use the ids to load segment data file for calc delete bitmap.
int64_t atomic_num_segments_before_flush =
_rowset_writer->get_atomic_num_segment();
- RETURN_NOT_OK(_do_flush(duration_ns));
+ RETURN_IF_ERROR(_do_flush(duration_ns));
int64_t atomic_num_segments_after_flush =
_rowset_writer->get_atomic_num_segment();
if (!_tablet_schema->is_partial_update()) {
- RETURN_NOT_OK(_generate_delete_bitmap(atomic_num_segments_before_flush,
-
atomic_num_segments_after_flush));
+
RETURN_IF_ERROR(_generate_delete_bitmap(atomic_num_segments_before_flush,
+
atomic_num_segments_after_flush));
}
DorisMetrics::instance()->memtable_flush_total->increment(1);
DorisMetrics::instance()->memtable_flush_duration_us->increment(duration_ns /
1000);
@@ -432,7 +432,7 @@ Status MemTable::_do_flush(int64_t& duration_ns) {
// Unfold variant column
unfold_variant_column(block);
}
- RETURN_NOT_OK(_rowset_writer->flush_single_memtable(&block, &_flush_size));
+ RETURN_IF_ERROR(_rowset_writer->flush_single_memtable(&block,
&_flush_size));
return Status::OK();
}
diff --git a/be/src/olap/merger.cpp b/be/src/olap/merger.cpp
index 437d9ba9eb..07a3d3d2cb 100644
--- a/be/src/olap/merger.cpp
+++ b/be/src/olap/merger.cpp
@@ -89,14 +89,14 @@ Status Merger::vmerge_rowsets(TabletSharedPtr tablet,
ReaderType reader_type,
reader_params.return_columns.resize(cur_tablet_schema->num_columns());
std::iota(reader_params.return_columns.begin(),
reader_params.return_columns.end(), 0);
reader_params.origin_return_columns = &reader_params.return_columns;
- RETURN_NOT_OK(reader.init(reader_params));
+ RETURN_IF_ERROR(reader.init(reader_params));
if (reader_params.record_rowids) {
stats_output->rowid_conversion->set_dst_rowset_id(dst_rowset_writer->rowset_id());
// init segment rowid map for rowid conversion
std::vector<uint32_t> segment_num_rows;
for (auto& rs_reader : reader_params.rs_readers) {
- RETURN_NOT_OK(rs_reader->get_segment_num_rows(&segment_num_rows));
+
RETURN_IF_ERROR(rs_reader->get_segment_num_rows(&segment_num_rows));
stats_output->rowid_conversion->init_segment_map(rs_reader->rowset()->rowset_id(),
segment_num_rows);
}
@@ -107,10 +107,10 @@ Status Merger::vmerge_rowsets(TabletSharedPtr tablet,
ReaderType reader_type,
bool eof = false;
while (!eof && !StorageEngine::instance()->stopped()) {
// Read one block from block reader
- RETURN_NOT_OK_LOG(
+ RETURN_NOT_OK_STATUS_WITH_WARN(
reader.next_block_with_aggregation(&block, &eof),
"failed to read next block when merging rowsets of tablet " +
tablet->full_name());
- RETURN_NOT_OK_LOG(
+ RETURN_NOT_OK_STATUS_WITH_WARN(
dst_rowset_writer->add_block(&block),
"failed to write block when merging rowsets of tablet " +
tablet->full_name());
@@ -135,7 +135,7 @@ Status Merger::vmerge_rowsets(TabletSharedPtr tablet,
ReaderType reader_type,
stats_output->filtered_rows = reader.filtered_rows();
}
- RETURN_NOT_OK_LOG(
+ RETURN_NOT_OK_STATUS_WITH_WARN(
dst_rowset_writer->flush(),
"failed to flush rowset when merging rowsets of tablet " +
tablet->full_name());
@@ -219,14 +219,14 @@ Status Merger::vertical_compact_one_group(
reader_params.return_columns = column_group;
reader_params.origin_return_columns = &reader_params.return_columns;
- RETURN_NOT_OK(reader.init(reader_params));
+ RETURN_IF_ERROR(reader.init(reader_params));
if (reader_params.record_rowids) {
stats_output->rowid_conversion->set_dst_rowset_id(dst_rowset_writer->rowset_id());
// init segment rowid map for rowid conversion
std::vector<uint32_t> segment_num_rows;
for (auto& rs_reader : reader_params.rs_readers) {
- RETURN_NOT_OK(rs_reader->get_segment_num_rows(&segment_num_rows));
+
RETURN_IF_ERROR(rs_reader->get_segment_num_rows(&segment_num_rows));
stats_output->rowid_conversion->init_segment_map(rs_reader->rowset()->rowset_id(),
segment_num_rows);
}
@@ -237,10 +237,10 @@ Status Merger::vertical_compact_one_group(
bool eof = false;
while (!eof && !StorageEngine::instance()->stopped()) {
// Read one block from block reader
- RETURN_NOT_OK_LOG(
+ RETURN_NOT_OK_STATUS_WITH_WARN(
reader.next_block_with_aggregation(&block, &eof),
"failed to read next block when merging rowsets of tablet " +
tablet->full_name());
- RETURN_NOT_OK_LOG(
+ RETURN_NOT_OK_STATUS_WITH_WARN(
dst_rowset_writer->add_columns(&block, column_group, is_key,
max_rows_per_segment),
"failed to write block when merging rowsets of tablet " +
tablet->full_name());
@@ -285,13 +285,13 @@ Status Merger::vertical_compact_one_group(TabletSharedPtr
tablet, ReaderType rea
bool eof = false;
while (!eof && !StorageEngine::instance()->stopped()) {
// Read one block from block reader
- RETURN_NOT_OK_LOG(
+ RETURN_NOT_OK_STATUS_WITH_WARN(
src_block_reader.next_block_with_aggregation(&block, &eof),
"failed to read next block when merging rowsets of tablet " +
tablet->full_name());
if (!block.rows()) {
break;
}
- RETURN_NOT_OK_LOG(
+ RETURN_NOT_OK_STATUS_WITH_WARN(
dst_segment_writer.append_block(&block, 0, block.rows()),
"failed to write block when merging rowsets of tablet " +
tablet->full_name());
diff --git a/be/src/olap/olap_define.h b/be/src/olap/olap_define.h
index ecfceda960..42f8dfe83b 100644
--- a/be/src/olap/olap_define.h
+++ b/be/src/olap/olap_define.h
@@ -148,35 +148,6 @@ const std::string ROWSET_ID_PREFIX = "s_";
const std::string REMOTE_ROWSET_GC_PREFIX = "gc_";
const std::string REMOTE_TABLET_GC_PREFIX = "tgc_";
-#if defined(__GNUC__)
-#define OLAP_LIKELY(x) __builtin_expect((x), 1)
-#define OLAP_UNLIKELY(x) __builtin_expect((x), 0)
-#else
-#define OLAP_LIKELY(x)
-#define OLAP_UNLIKELY(x)
-#endif
-
-#ifndef RETURN_NOT_OK
-#define RETURN_NOT_OK(s) \
- do { \
- Status _s = (s); \
- if (OLAP_UNLIKELY(!_s.ok())) { \
- return _s; \
- } \
- } while (0);
-#endif
-
-#ifndef RETURN_NOT_OK_LOG
-#define RETURN_NOT_OK_LOG(s, msg) \
- do { \
- Status _s = (s); \
- if (OLAP_UNLIKELY(!_s)) { \
- LOG(WARNING) << (msg) << "[res=" << _s << "]"; \
- return _s; \
- } \
- } while (0);
-#endif
-
// Declare copy constructor and equal operator as private
#ifndef DISALLOW_COPY_AND_ASSIGN
#define DISALLOW_COPY_AND_ASSIGN(type_t) \
diff --git a/be/src/olap/options.cpp b/be/src/olap/options.cpp
index 337ff65ba4..3d60631f0f 100644
--- a/be/src/olap/options.cpp
+++ b/be/src/olap/options.cpp
@@ -36,6 +36,7 @@
#include "olap/olap_define.h"
#include "olap/utils.h"
#include "util/path_util.h"
+#include "util/string_util.h"
namespace doris {
using namespace ErrorCode;
@@ -54,11 +55,11 @@ static std::string CACHE_TOTAL_SIZE = "total_size";
static std::string CACHE_QUERY_LIMIT_SIZE = "query_limit";
// TODO: should be a general util method
-static std::string to_upper(const std::string& str) {
- std::string out = str;
- std::transform(out.begin(), out.end(), out.begin(), [](auto c) { return
std::toupper(c); });
- return out;
-}
+// static std::string to_upper(const std::string& str) {
+// std::string out = str;
+// std::transform(out.begin(), out.end(), out.begin(), [](auto c) { return
std::toupper(c); });
+// return out;
+// }
// Currently, both of three following formats are supported(see be.conf),
remote cache is the
// local cache path for remote storage.
diff --git a/be/src/olap/push_handler.cpp b/be/src/olap/push_handler.cpp
index 6734d96c5c..482ffe578d 100644
--- a/be/src/olap/push_handler.cpp
+++ b/be/src/olap/push_handler.cpp
@@ -119,7 +119,7 @@ Status PushHandler::_do_streaming_ingestion(TabletSharedPtr
tablet, const TPushR
load_id.set_lo(0);
{
std::lock_guard<std::mutex> push_lock(tablet->get_push_lock());
- RETURN_NOT_OK(StorageEngine::instance()->txn_manager()->prepare_txn(
+ RETURN_IF_ERROR(StorageEngine::instance()->txn_manager()->prepare_txn(
request.partition_id, tablet, request.transaction_id,
load_id));
}
diff --git a/be/src/olap/reader.cpp b/be/src/olap/reader.cpp
index 53d06a2f32..90906db22c 100644
--- a/be/src/olap/reader.cpp
+++ b/be/src/olap/reader.cpp
@@ -632,7 +632,7 @@ Status TabletReader::init_reader_params_and_create_block(
for (auto& rowset : input_rowsets) {
RowsetReaderSharedPtr rs_reader;
- RETURN_NOT_OK(rowset->create_reader(&rs_reader));
+ RETURN_IF_ERROR(rowset->create_reader(&rs_reader));
reader_params->rs_readers.push_back(std::move(rs_reader));
}
diff --git a/be/src/olap/row_cursor.cpp b/be/src/olap/row_cursor.cpp
index ed77220e87..7254c1df18 100644
--- a/be/src/olap/row_cursor.cpp
+++ b/be/src/olap/row_cursor.cpp
@@ -105,7 +105,7 @@ Status RowCursor::_init_scan_key(TabletSchemaSPtr schema,
}
// variable_len for null bytes
- RETURN_NOT_OK(_alloc_buf());
+ RETURN_IF_ERROR(_alloc_buf());
char* fixed_ptr = _fixed_buf;
char* variable_ptr = _variable_buf;
char** long_text_ptr = _long_text_buf;
@@ -156,7 +156,7 @@ Status RowCursor::init(TabletSchemaSPtr schema, size_t
column_count) {
for (size_t i = 0; i < column_count; ++i) {
columns.push_back(i);
}
- RETURN_NOT_OK(_init(schema->columns(), columns));
+ RETURN_IF_ERROR(_init(schema->columns(), columns));
return Status::OK();
}
@@ -172,12 +172,12 @@ Status RowCursor::init(const std::vector<TabletColumn>&
schema, size_t column_co
for (size_t i = 0; i < column_count; ++i) {
columns.push_back(i);
}
- RETURN_NOT_OK(_init(schema, columns));
+ RETURN_IF_ERROR(_init(schema, columns));
return Status::OK();
}
Status RowCursor::init(TabletSchemaSPtr schema, const std::vector<uint32_t>&
columns) {
- RETURN_NOT_OK(_init(schema->columns(), columns));
+ RETURN_IF_ERROR(_init(schema->columns(), columns));
return Status::OK();
}
@@ -195,7 +195,7 @@ Status RowCursor::init_scan_key(TabletSchemaSPtr schema,
std::vector<uint32_t> columns(scan_key_size);
std::iota(columns.begin(), columns.end(), 0);
- RETURN_NOT_OK(_init(schema->columns(), columns));
+ RETURN_IF_ERROR(_init(schema->columns(), columns));
return _init_scan_key(schema, scan_keys);
}
@@ -209,7 +209,7 @@ Status RowCursor::init_scan_key(TabletSchemaSPtr schema,
const std::vector<std::
columns.push_back(i);
}
- RETURN_NOT_OK(_init(shared_schema, columns));
+ RETURN_IF_ERROR(_init(shared_schema, columns));
return _init_scan_key(schema, scan_keys);
}
@@ -222,7 +222,7 @@ Status
RowCursor::allocate_memory_for_string_type(TabletSchemaSPtr schema) {
return Status::OK();
}
DCHECK(_variable_buf == nullptr) << "allocate memory twice";
- RETURN_NOT_OK(_alloc_buf());
+ RETURN_IF_ERROR(_alloc_buf());
// init slice of char, varchar, hll type
char* fixed_ptr = _fixed_buf;
char* variable_ptr = _variable_buf;
diff --git a/be/src/olap/rowset/CMakeLists.txt
b/be/src/olap/rowset/CMakeLists.txt
index 68b3428a72..6680bb4f3e 100644
--- a/be/src/olap/rowset/CMakeLists.txt
+++ b/be/src/olap/rowset/CMakeLists.txt
@@ -31,4 +31,7 @@ add_library(Rowset STATIC
beta_rowset_writer.cpp
vertical_beta_rowset_writer.cpp
segcompaction.cpp
- rowset_tree.cpp)
+ rowset_tree.cpp
+)
+
+pch_reuse(Rowset)
diff --git a/be/src/olap/rowset/beta_rowset_reader.cpp
b/be/src/olap/rowset/beta_rowset_reader.cpp
index dcc2dd03f1..670f48d3d3 100644
--- a/be/src/olap/rowset/beta_rowset_reader.cpp
+++ b/be/src/olap/rowset/beta_rowset_reader.cpp
@@ -75,7 +75,7 @@ Status
BetaRowsetReader::get_segment_iterators(RowsetReaderContext* read_context
std::vector<RowwiseIteratorUPtr>* out_iters,
const std::pair<int, int>&
segment_offset,
bool use_cache) {
- RETURN_NOT_OK(_rowset->load());
+ RETURN_IF_ERROR(_rowset->load());
_context = read_context;
if (_context->stats != nullptr) {
// schema change/compaction should use owned_stats
@@ -193,8 +193,8 @@ Status
BetaRowsetReader::get_segment_iterators(RowsetReaderContext* read_context
// load segments
// use cache is true when do vertica compaction
bool should_use_cache = use_cache || read_context->reader_type ==
ReaderType::READER_QUERY;
- RETURN_NOT_OK(SegmentLoader::instance()->load_segments(_rowset,
&_segment_cache_handle,
- should_use_cache));
+ RETURN_IF_ERROR(SegmentLoader::instance()->load_segments(_rowset,
&_segment_cache_handle,
+
should_use_cache));
// create iterator for each segment
auto& segments = _segment_cache_handle.get_segments();
@@ -225,7 +225,7 @@ Status BetaRowsetReader::init(RowsetReaderContext*
read_context,
const std::pair<int, int>& segment_offset) {
_context = read_context;
std::vector<RowwiseIteratorUPtr> iterators;
- RETURN_NOT_OK(get_segment_iterators(_context, &iterators, segment_offset));
+ RETURN_IF_ERROR(get_segment_iterators(_context, &iterators,
segment_offset));
// merge or union segment iterator
if (read_context->need_ordered_result &&
_rowset->rowset_meta()->is_segments_overlapping()) {
diff --git a/be/src/olap/rowset/beta_rowset_writer.cpp
b/be/src/olap/rowset/beta_rowset_writer.cpp
index 224944d917..425f882d76 100644
--- a/be/src/olap/rowset/beta_rowset_writer.cpp
+++ b/be/src/olap/rowset/beta_rowset_writer.cpp
@@ -132,7 +132,7 @@ Status BetaRowsetWriter::add_block(const vectorized::Block*
block) {
return Status::OK();
}
if (UNLIKELY(_segment_writer == nullptr)) {
- RETURN_NOT_OK(_create_segment_writer(&_segment_writer, block));
+ RETURN_IF_ERROR(_create_segment_writer(&_segment_writer, block));
}
return _add_block(block, &_segment_writer);
}
@@ -171,7 +171,7 @@ Status
BetaRowsetWriter::_find_longest_consecutive_small_segment(
SegCompactionCandidatesSharedPtr segments) {
std::vector<segment_v2::SegmentSharedPtr> all_segments;
// subtract one to skip last (maybe active) segment
- RETURN_NOT_OK(_load_noncompacted_segments(&all_segments, _num_segment -
1));
+ RETURN_IF_ERROR(_load_noncompacted_segments(&all_segments, _num_segment -
1));
if (VLOG_DEBUG_IS_ON) {
vlog_buffer.clear();
@@ -192,7 +192,7 @@ Status
BetaRowsetWriter::_find_longest_consecutive_small_segment(
is_terminated_by_big = true;
break;
} else {
-
RETURN_NOT_OK(_rename_compacted_segment_plain(_segcompacted_point++));
+
RETURN_IF_ERROR(_rename_compacted_segment_plain(_segcompacted_point++));
}
} else {
let_big_terminate = true; // break if find a big after small
@@ -208,7 +208,7 @@ Status
BetaRowsetWriter::_find_longest_consecutive_small_segment(
}
if (s == 1) { // poor bachelor, let it go
VLOG_DEBUG << "only one candidate segment";
- RETURN_NOT_OK(_rename_compacted_segment_plain(_segcompacted_point++));
+
RETURN_IF_ERROR(_rename_compacted_segment_plain(_segcompacted_point++));
segments->clear();
return Status::OK();
}
@@ -237,7 +237,7 @@ Status BetaRowsetWriter::_rename_compacted_segments(int64_t
begin, int64_t end)
}
// rename inverted index files
- RETURN_NOT_OK(_rename_compacted_indices(begin, end, 0));
+ RETURN_IF_ERROR(_rename_compacted_indices(begin, end, 0));
_num_segcompacted++;
return Status::OK();
@@ -280,7 +280,7 @@ Status
BetaRowsetWriter::_rename_compacted_segment_plain(uint64_t seg_id) {
return Status::Error<ROWSET_RENAME_FILE_FAILED>();
}
// rename remaining inverted index files
- RETURN_NOT_OK(_rename_compacted_indices(-1, -1, seg_id));
+ RETURN_IF_ERROR(_rename_compacted_indices(-1, -1, seg_id));
++_num_segcompacted;
return Status::OK();
@@ -323,13 +323,13 @@ Status
BetaRowsetWriter::_get_segcompaction_candidates(SegCompactionCandidatesSh
VLOG_DEBUG << "segcompaction last few segments";
// currently we only rename remaining segments to reduce wait time
// so that transaction can be committed ASAP
- RETURN_NOT_OK(_load_noncompacted_segments(segments.get(),
_num_segment));
+ RETURN_IF_ERROR(_load_noncompacted_segments(segments.get(),
_num_segment));
for (int i = 0; i < segments->size(); ++i) {
-
RETURN_NOT_OK(_rename_compacted_segment_plain(_segcompacted_point++));
+
RETURN_IF_ERROR(_rename_compacted_segment_plain(_segcompacted_point++));
}
segments->clear();
} else {
- RETURN_NOT_OK(_find_longest_consecutive_small_segment(segments));
+ RETURN_IF_ERROR(_find_longest_consecutive_small_segment(segments));
}
return Status::OK();
}
@@ -415,8 +415,8 @@ Status BetaRowsetWriter::_add_block(const
vectorized::Block* block,
auto max_row_add =
(*segment_writer)->max_row_to_add(row_avg_size_in_bytes);
if (UNLIKELY(max_row_add < 1)) {
// no space for another single row, need flush now
- RETURN_NOT_OK(_flush_segment_writer(segment_writer));
- RETURN_NOT_OK(_create_segment_writer(segment_writer, block));
+ RETURN_IF_ERROR(_flush_segment_writer(segment_writer));
+ RETURN_IF_ERROR(_create_segment_writer(segment_writer, block));
max_row_add =
(*segment_writer)->max_row_to_add(row_avg_size_in_bytes);
DCHECK(max_row_add > 0);
}
@@ -436,7 +436,7 @@ Status BetaRowsetWriter::_add_block(const
vectorized::Block* block,
Status BetaRowsetWriter::add_rowset(RowsetSharedPtr rowset) {
assert(rowset->rowset_meta()->rowset_type() == BETA_ROWSET);
- RETURN_NOT_OK(rowset->link_files_to(_context.rowset_dir,
_context.rowset_id));
+ RETURN_IF_ERROR(rowset->link_files_to(_context.rowset_dir,
_context.rowset_id));
_num_rows_written += rowset->num_rows();
_total_data_size += rowset->rowset_meta()->data_disk_size();
_total_index_size += rowset->rowset_meta()->index_disk_size();
@@ -457,7 +457,7 @@ Status
BetaRowsetWriter::add_rowset_for_linked_schema_change(RowsetSharedPtr row
Status BetaRowsetWriter::flush() {
if (_segment_writer != nullptr) {
- RETURN_NOT_OK(_flush_segment_writer(&_segment_writer));
+ RETURN_IF_ERROR(_flush_segment_writer(&_segment_writer));
}
return Status::OK();
}
@@ -468,10 +468,10 @@ Status BetaRowsetWriter::flush_single_memtable(const
vectorized::Block* block, i
}
std::unique_ptr<segment_v2::SegmentWriter> writer;
- RETURN_NOT_OK(_create_segment_writer(&writer, block));
- RETURN_NOT_OK(_add_block(block, &writer));
- RETURN_NOT_OK(_flush_segment_writer(&writer, flush_size));
- RETURN_NOT_OK(_segcompaction_if_necessary());
+ RETURN_IF_ERROR(_create_segment_writer(&writer, block));
+ RETURN_IF_ERROR(_add_block(block, &writer));
+ RETURN_IF_ERROR(_flush_segment_writer(&writer, flush_size));
+ RETURN_IF_ERROR(_segcompaction_if_necessary());
return Status::OK();
}
diff --git a/be/src/olap/rowset/rowset.cpp b/be/src/olap/rowset/rowset.cpp
index 699d384b5f..eefd6a01b2 100644
--- a/be/src/olap/rowset/rowset.cpp
+++ b/be/src/olap/rowset/rowset.cpp
@@ -51,8 +51,8 @@ Status Rowset::load(bool use_cache) {
// after lock, if rowset state is ROWSET_UNLOADING, it is ok to return
if (_rowset_state_machine.rowset_state() == ROWSET_UNLOADED) {
// first do load, then change the state
- RETURN_NOT_OK(do_load(use_cache));
- RETURN_NOT_OK(_rowset_state_machine.on_load());
+ RETURN_IF_ERROR(do_load(use_cache));
+ RETURN_IF_ERROR(_rowset_state_machine.on_load());
}
}
// load is done
diff --git a/be/src/olap/rowset/segcompaction.cpp
b/be/src/olap/rowset/segcompaction.cpp
index 8eb60eae7f..a8187e6c52 100644
--- a/be/src/olap/rowset/segcompaction.cpp
+++ b/be/src/olap/rowset/segcompaction.cpp
@@ -127,8 +127,8 @@ Status
SegcompactionWorker::_delete_original_segments(uint32_t begin, uint32_t e
// Even if an error is encountered, these files that have not been
cleaned up
// will be cleaned up by the GC background. So here we only print the
error
// message when we encounter an error.
- RETURN_NOT_OK_LOG(fs->delete_file(seg_path),
- strings::Substitute("Failed to delete file=$0",
seg_path));
+ RETURN_NOT_OK_STATUS_WITH_WARN(fs->delete_file(seg_path),
+ strings::Substitute("Failed to delete
file=$0", seg_path));
// Delete inverted index files
for (auto column : schema->columns()) {
if (schema->has_inverted_index(column.unique_id())) {
@@ -136,8 +136,9 @@ Status
SegcompactionWorker::_delete_original_segments(uint32_t begin, uint32_t e
auto idx_path =
InvertedIndexDescriptor::inverted_index_file_path(
ctx.rowset_dir, ctx.rowset_id, i, index_id);
VLOG_DEBUG << "segcompaction index. delete file " << idx_path;
- RETURN_NOT_OK_LOG(fs->delete_file(idx_path),
- strings::Substitute("Failed to delete
file=$0", idx_path));
+ RETURN_NOT_OK_STATUS_WITH_WARN(
+ fs->delete_file(idx_path),
+ strings::Substitute("Failed to delete file=$0",
idx_path));
// Erase the origin index file cache
InvertedIndexSearcherCache::instance()->erase(idx_path);
}
@@ -252,21 +253,22 @@ Status
SegcompactionWorker::_do_compact_segments(SegCompactionCandidatesSharedPt
}
/* check row num after merge/aggregation */
- RETURN_NOT_OK_LOG(_check_correctness(key_reader_stats, key_merger_stats,
begin, end),
- "check correctness failed");
+ RETURN_NOT_OK_STATUS_WITH_WARN(
+ _check_correctness(key_reader_stats, key_merger_stats, begin, end),
+ "check correctness failed");
{
std::lock_guard<std::mutex> lock(_writer->_segid_statistics_map_mutex);
_writer->_clear_statistics_for_deleting_segments_unsafe(begin, end);
}
- RETURN_NOT_OK(
+ RETURN_IF_ERROR(
_writer->flush_segment_writer_for_segcompaction(&writer,
total_index_size, key_bounds));
if (_file_writer != nullptr) {
_file_writer->close();
}
- RETURN_NOT_OK(_delete_original_segments(begin, end));
- RETURN_NOT_OK(_writer->_rename_compacted_segments(begin, end));
+ RETURN_IF_ERROR(_delete_original_segments(begin, end));
+ RETURN_IF_ERROR(_writer->_rename_compacted_segments(begin, end));
if (VLOG_DEBUG_IS_ON) {
_writer->vlog_buffer.clear();
diff --git a/be/src/olap/schema_change.cpp b/be/src/olap/schema_change.cpp
index 3bf822c7c2..3ceebd22d3 100644
--- a/be/src/olap/schema_change.cpp
+++ b/be/src/olap/schema_change.cpp
@@ -661,7 +661,7 @@ Status
SchemaChangeForInvertedIndex::process(RowsetReaderSharedPtr rowset_reader
// load segments
SegmentCacheHandle segment_cache_handle;
- RETURN_NOT_OK(SegmentLoader::instance()->load_segments(
+ RETURN_IF_ERROR(SegmentLoader::instance()->load_segments(
std::static_pointer_cast<BetaRowset>(rowset_reader->rowset()),
&segment_cache_handle,
false));
@@ -731,7 +731,7 @@ Status
SchemaChangeForInvertedIndex::process(RowsetReaderSharedPtr rowset_reader
if (res.is<END_OF_FILE>()) {
break;
}
- RETURN_NOT_OK_LOG(
+ RETURN_NOT_OK_STATUS_WITH_WARN(
res, "failed to read next block when schema change for
inverted index.");
}
@@ -1548,8 +1548,8 @@ Status SchemaChangeHandler::_get_versions_to_be_changed(
}
*max_rowset = rowset;
- RETURN_NOT_OK(base_tablet->capture_consistent_versions(Version(0,
rowset->version().second),
-
versions_to_be_changed));
+ RETURN_IF_ERROR(base_tablet->capture_consistent_versions(Version(0,
rowset->version().second),
+
versions_to_be_changed));
return Status::OK();
}
diff --git a/be/src/olap/segment_loader.cpp b/be/src/olap/segment_loader.cpp
index f244f2bb71..3d61ef05db 100644
--- a/be/src/olap/segment_loader.cpp
+++ b/be/src/olap/segment_loader.cpp
@@ -74,7 +74,7 @@ Status SegmentLoader::load_segments(const
BetaRowsetSharedPtr& rowset,
cache_handle->owned = !use_cache;
std::vector<segment_v2::SegmentSharedPtr> segments;
- RETURN_NOT_OK(rowset->load_segments(&segments));
+ RETURN_IF_ERROR(rowset->load_segments(&segments));
if (use_cache) {
// memory of SegmentLoader::CacheValue will be handled by SegmentLoader
diff --git a/be/src/olap/segment_loader.h b/be/src/olap/segment_loader.h
index 292b5df988..9e6a8af82a 100644
--- a/be/src/olap/segment_loader.h
+++ b/be/src/olap/segment_loader.h
@@ -46,7 +46,7 @@ class BetaRowset;
// the segments of a specified rowset:
//
// SegmentCacheHandle cache_handle;
-// RETURN_NOT_OK(SegmentCache::instance()->load_segments(_rowset,
&cache_handle));
+// RETURN_IF_ERROR(SegmentCache::instance()->load_segments(_rowset,
&cache_handle));
// for (auto& seg_ptr : cache_handle.value()->segments) {
// ... visit segment ...
// }
diff --git a/be/src/olap/snapshot_manager.cpp b/be/src/olap/snapshot_manager.cpp
index d1ef6a96b5..6830d86767 100644
--- a/be/src/olap/snapshot_manager.cpp
+++ b/be/src/olap/snapshot_manager.cpp
@@ -178,8 +178,8 @@ Status SnapshotManager::convert_rowset_ids(const
std::string& clone_dir, int64_t
if (!visible_rowset.has_resource_id()) {
// src be local rowset
RowsetId rowset_id = StorageEngine::instance()->next_rowset_id();
- RETURN_NOT_OK(_rename_rowset_id(visible_rowset, clone_dir,
tablet_schema, rowset_id,
- rowset_meta));
+ RETURN_IF_ERROR(_rename_rowset_id(visible_rowset, clone_dir,
tablet_schema, rowset_id,
+ rowset_meta));
RowsetId src_rs_id;
if (visible_rowset.rowset_id() > 0) {
src_rs_id.init(visible_rowset.rowset_id());
@@ -206,8 +206,8 @@ Status SnapshotManager::convert_rowset_ids(const
std::string& clone_dir, int64_t
if (!stale_rowset.has_resource_id()) {
// src be local rowset
RowsetId rowset_id = StorageEngine::instance()->next_rowset_id();
- RETURN_NOT_OK(_rename_rowset_id(stale_rowset, clone_dir,
tablet_schema, rowset_id,
- rowset_meta));
+ RETURN_IF_ERROR(_rename_rowset_id(stale_rowset, clone_dir,
tablet_schema, rowset_id,
+ rowset_meta));
RowsetId src_rs_id;
if (stale_rowset.rowset_id() > 0) {
src_rs_id.init(stale_rowset.rowset_id());
@@ -253,12 +253,12 @@ Status SnapshotManager::_rename_rowset_id(const
RowsetMetaPB& rs_meta_pb,
RowsetMetaSharedPtr rowset_meta(new RowsetMeta());
rowset_meta->init_from_pb(rs_meta_pb);
RowsetSharedPtr org_rowset;
- RETURN_NOT_OK(
+ RETURN_IF_ERROR(
RowsetFactory::create_rowset(tablet_schema, new_tablet_path,
rowset_meta, &org_rowset));
// do not use cache to load index
// because the index file may conflict
// and the cached fd may be invalid
- RETURN_NOT_OK(org_rowset->load(false));
+ RETURN_IF_ERROR(org_rowset->load(false));
RowsetMetaSharedPtr org_rowset_meta = org_rowset->rowset_meta();
RowsetWriterContext context;
context.rowset_id = rowset_id;
@@ -276,7 +276,7 @@ Status SnapshotManager::_rename_rowset_id(const
RowsetMetaPB& rs_meta_pb,
context.segments_overlap = rowset_meta->segments_overlap();
std::unique_ptr<RowsetWriter> rs_writer;
- RETURN_NOT_OK(RowsetFactory::create_rowset_writer(context, false,
&rs_writer));
+ RETURN_IF_ERROR(RowsetFactory::create_rowset_writer(context, false,
&rs_writer));
res = rs_writer->add_rowset(org_rowset);
if (!res.ok()) {
@@ -289,7 +289,7 @@ Status SnapshotManager::_rename_rowset_id(const
RowsetMetaPB& rs_meta_pb,
LOG(WARNING) << "failed to build rowset when rename rowset id";
return Status::Error<MEM_ALLOC_FAILED>();
}
- RETURN_NOT_OK(new_rowset->load(false));
+ RETURN_IF_ERROR(new_rowset->load(false));
new_rowset->rowset_meta()->to_rowset_pb(new_rs_meta_pb);
org_rowset->remove();
return Status::OK();
@@ -342,7 +342,7 @@ Status SnapshotManager::_link_index_and_data_files(
const std::vector<RowsetSharedPtr>& consistent_rowsets) {
Status res = Status::OK();
for (auto& rs : consistent_rowsets) {
- RETURN_NOT_OK(rs->link_files_to(schema_hash_path, rs->rowset_id()));
+ RETURN_IF_ERROR(rs->link_files_to(schema_hash_path, rs->rowset_id()));
}
return res;
}
diff --git a/be/src/olap/storage_engine.cpp b/be/src/olap/storage_engine.cpp
index 4015d6f10c..79bb7c5363 100644
--- a/be/src/olap/storage_engine.cpp
+++ b/be/src/olap/storage_engine.cpp
@@ -628,8 +628,8 @@ Status StorageEngine::start_trash_sweep(double* usage, bool
ignore_guard) {
const double guard_space =
ignore_guard ? 0 : config::storage_flood_stage_usage_percent /
100.0 * 0.9;
std::vector<DataDirInfo> data_dir_infos;
- RETURN_NOT_OK_LOG(get_all_data_dir_info(&data_dir_infos, false),
- "failed to get root path stat info when sweep trash.")
+ RETURN_NOT_OK_STATUS_WITH_WARN(get_all_data_dir_info(&data_dir_infos,
false),
+ "failed to get root path stat info when
sweep trash.")
std::sort(data_dir_infos.begin(), data_dir_infos.end(),
DataDirInfoLessAvailability());
time_t now = time(nullptr); //获取UTC时间
diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp
index d3b010d787..9b4e94bae2 100644
--- a/be/src/olap/tablet.cpp
+++ b/be/src/olap/tablet.cpp
@@ -401,9 +401,9 @@ Status Tablet::add_rowset(RowsetSharedPtr rowset) {
return Status::OK();
}
// Otherwise, the version should be not contained in any existing rowset.
- RETURN_NOT_OK(_contains_version(rowset->version()));
+ RETURN_IF_ERROR(_contains_version(rowset->version()));
- RETURN_NOT_OK(_tablet_meta->add_rs_meta(rowset->rowset_meta()));
+ RETURN_IF_ERROR(_tablet_meta->add_rs_meta(rowset->rowset_meta()));
_rs_version_map[rowset->version()] = rowset;
_timestamped_version_tracker.add_version(rowset->version());
@@ -638,9 +638,9 @@ Status Tablet::add_inc_rowset(const RowsetSharedPtr&
rowset) {
if (_contains_rowset(rowset->rowset_id())) {
return Status::OK();
}
- RETURN_NOT_OK(_contains_version(rowset->version()));
+ RETURN_IF_ERROR(_contains_version(rowset->version()));
- RETURN_NOT_OK(_tablet_meta->add_rs_meta(rowset->rowset_meta()));
+ RETURN_IF_ERROR(_tablet_meta->add_rs_meta(rowset->rowset_meta()));
_rs_version_map[rowset->version()] = rowset;
// Update rowset tree
@@ -883,8 +883,8 @@ void Tablet::acquire_version_and_rowsets(
Status Tablet::capture_consistent_rowsets(const Version& spec_version,
std::vector<RowsetSharedPtr>*
rowsets) const {
std::vector<Version> version_path;
- RETURN_NOT_OK(capture_consistent_versions(spec_version, &version_path));
- RETURN_NOT_OK(_capture_consistent_rowsets_unlocked(version_path, rowsets));
+ RETURN_IF_ERROR(capture_consistent_versions(spec_version, &version_path));
+ RETURN_IF_ERROR(_capture_consistent_rowsets_unlocked(version_path,
rowsets));
return Status::OK();
}
@@ -922,8 +922,8 @@ Status Tablet::_capture_consistent_rowsets_unlocked(const
std::vector<Version>&
Status Tablet::capture_rs_readers(const Version& spec_version,
std::vector<RowsetReaderSharedPtr>*
rs_readers) const {
std::vector<Version> version_path;
- RETURN_NOT_OK(capture_consistent_versions(spec_version, &version_path));
- RETURN_NOT_OK(capture_rs_readers(version_path, rs_readers));
+ RETURN_IF_ERROR(capture_consistent_versions(spec_version, &version_path));
+ RETURN_IF_ERROR(capture_rs_readers(version_path, rs_readers));
return Status::OK();
}
@@ -2561,7 +2561,7 @@ Status Tablet::lookup_row_key(const Slice& encoded_key,
bool with_seq_col,
continue;
}
SegmentCacheHandle segment_cache_handle;
- RETURN_NOT_OK(SegmentLoader::instance()->load_segments(
+ RETURN_IF_ERROR(SegmentLoader::instance()->load_segments(
std::static_pointer_cast<BetaRowset>(rs.first),
&segment_cache_handle, true));
auto& segments = segment_cache_handle.get_segments();
DCHECK_GT(segments.size(), rs.second);
diff --git a/be/src/olap/tablet_manager.cpp b/be/src/olap/tablet_manager.cpp
index e236c0668f..9ede6f0d73 100644
--- a/be/src/olap/tablet_manager.cpp
+++ b/be/src/olap/tablet_manager.cpp
@@ -205,10 +205,11 @@ Status
TabletManager::_add_tablet_to_map_unlocked(TTabletId tablet_id,
// If the new tablet is fresher than the existing one, then replace
// the existing tablet with the new one.
// Use default replica_id to ignore whether replica_id is match when
drop tablet.
- RETURN_NOT_OK_LOG(_drop_tablet_unlocked(tablet_id, /* replica_id */ 0,
keep_files, false),
- strings::Substitute("failed to drop old tablet when
add new tablet. "
- "tablet_id=$0",
- tablet_id));
+ RETURN_NOT_OK_STATUS_WITH_WARN(
+ _drop_tablet_unlocked(tablet_id, /* replica_id */ 0,
keep_files, false),
+ strings::Substitute("failed to drop old tablet when add new
tablet. "
+ "tablet_id=$0",
+ tablet_id));
}
// Register tablet into DataDir, so that we can manage tablet from
// the perspective of root path.
@@ -822,12 +823,14 @@ Status TabletManager::load_tablet_from_meta(DataDir*
data_dir, TTabletId tablet_
return Status::Error<TABLE_INDEX_VALIDATE_ERROR>();
}
- RETURN_NOT_OK_LOG(tablet->init(),
- strings::Substitute("tablet init failed. tablet=$0",
tablet->full_name()));
+ RETURN_NOT_OK_STATUS_WITH_WARN(
+ tablet->init(),
+ strings::Substitute("tablet init failed. tablet=$0",
tablet->full_name()));
std::lock_guard<std::shared_mutex>
wrlock(_get_tablets_shard_lock(tablet_id));
- RETURN_NOT_OK_LOG(_add_tablet_unlocked(tablet_id, tablet, update_meta,
force),
- strings::Substitute("fail to add tablet. tablet=$0",
tablet->full_name()));
+ RETURN_NOT_OK_STATUS_WITH_WARN(
+ _add_tablet_unlocked(tablet_id, tablet, update_meta, force),
+ strings::Substitute("fail to add tablet. tablet=$0",
tablet->full_name()));
return Status::OK();
}
@@ -867,9 +870,11 @@ Status TabletManager::load_tablet_from_dir(DataDir* store,
TTabletId tablet_id,
tablet_meta->set_tablet_uid(TabletUid::gen_uid());
std::string meta_binary;
tablet_meta->serialize(&meta_binary);
- RETURN_NOT_OK_LOG(load_tablet_from_meta(store, tablet_id, schema_hash,
meta_binary, true, force,
- restore, true),
- strings::Substitute("fail to load tablet.
header_path=$0", header_path));
+ RETURN_NOT_OK_STATUS_WITH_WARN(
+ load_tablet_from_meta(store, tablet_id, schema_hash, meta_binary,
true, force, restore,
+ true),
+ strings::Substitute("fail to load tablet. header_path=$0",
header_path));
+
return Status::OK();
}
@@ -1196,10 +1201,10 @@ Status
TabletManager::_create_tablet_meta_unlocked(const TCreateTabletReq& reque
// We generate a new tablet_uid for this new tablet.
uint64_t shard_id = 0;
- RETURN_NOT_OK_LOG(store->get_shard(&shard_id), "fail to get root path
shard");
+ RETURN_NOT_OK_STATUS_WITH_WARN(store->get_shard(&shard_id), "fail to get
root path shard");
Status res = TabletMeta::create(request, TabletUid::gen_uid(), shard_id,
next_unique_id,
col_idx_to_unique_id, tablet_meta);
- RETURN_NOT_OK(res);
+ RETURN_IF_ERROR(res);
if (request.__isset.storage_format) {
if (request.storage_format == TStorageFormat::DEFAULT) {
(*tablet_meta)
diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp
index 05ce5aaf84..1294720fa2 100644
--- a/be/src/olap/tablet_meta.cpp
+++ b/be/src/olap/tablet_meta.cpp
@@ -402,7 +402,7 @@ Status TabletMeta::_save_meta(DataDir* data_dir) {
<< " tablet=" << full_name() << " _tablet_uid=" <<
_tablet_uid.to_string();
}
string meta_binary;
- RETURN_NOT_OK(serialize(&meta_binary));
+ RETURN_IF_ERROR(serialize(&meta_binary));
Status status = TabletMetaManager::save(data_dir, tablet_id(),
schema_hash(), meta_binary);
if (!status.ok()) {
LOG(FATAL) << "fail to save tablet_meta. status=" << status << ",
tablet_id=" << tablet_id()
diff --git a/be/src/olap/task/engine_checksum_task.cpp
b/be/src/olap/task/engine_checksum_task.cpp
index fd98691415..1abc0a7e92 100644
--- a/be/src/olap/task/engine_checksum_task.cpp
+++ b/be/src/olap/task/engine_checksum_task.cpp
@@ -84,8 +84,8 @@ Status EngineChecksumTask::_compute_checksum() {
vectorized::BlockReader reader;
TabletReader::ReaderParams reader_params;
vectorized::Block block;
- RETURN_NOT_OK(TabletReader::init_reader_params_and_create_block(
- tablet, READER_CHECKSUM, input_rowsets, &reader_params, &block))
+ RETURN_IF_ERROR(TabletReader::init_reader_params_and_create_block(
+ tablet, READER_CHECKSUM, input_rowsets, &reader_params, &block));
auto res = reader.init(reader_params);
if (!res.ok()) {
diff --git a/be/src/olap/txn_manager.cpp b/be/src/olap/txn_manager.cpp
index 5df068630b..ab6af8e293 100644
--- a/be/src/olap/txn_manager.cpp
+++ b/be/src/olap/txn_manager.cpp
@@ -337,7 +337,7 @@ Status TxnManager::publish_txn(OlapMeta* meta, TPartitionId
partition_id,
rowset_writer.get()));
if (rowset_ptr->tablet_schema()->is_partial_update()) {
// build rowset writer and merge transient rowset
- RETURN_NOT_OK(rowset_writer->flush());
+ RETURN_IF_ERROR(rowset_writer->flush());
RowsetSharedPtr transient_rowset =
rowset_writer->build();
rowset_ptr->merge_rowset_meta(transient_rowset->rowset_meta());
@@ -396,7 +396,7 @@ Status
TxnManager::_create_transient_rowset_writer(std::shared_ptr<Tablet> table
context.tablet_id = tablet->table_id();
context.tablet = tablet;
context.is_direct_write = true;
- RETURN_NOT_OK(tablet->create_transient_rowset_writer(context, rowset_id,
rowset_writer));
+ RETURN_IF_ERROR(tablet->create_transient_rowset_writer(context, rowset_id,
rowset_writer));
(*rowset_writer)->set_segment_start_id(num_segments_ori);
return Status::OK();
}
diff --git a/be/src/olap/types.h b/be/src/olap/types.h
index 698b6548f7..a079d96b20 100644
--- a/be/src/olap/types.h
+++ b/be/src/olap/types.h
@@ -914,7 +914,7 @@ struct FieldTypeTraits<FieldType::OLAP_FIELD_TYPE_LARGEINT>
current += snprintf(current, end - current, "%" PRIu64,
prefix);
current += snprintf(current, end - current, "%.19" PRIu64,
middle);
current += snprintf(current, end - current, "%.19" PRIu64,
suffix);
- } else if (OLAP_LIKELY(middle > 0)) {
+ } else if (LIKELY(middle > 0)) {
current += snprintf(current, end - current, "%" PRIu64,
middle);
current += snprintf(current, end - current, "%.19" PRIu64,
suffix);
} else {
diff --git a/be/src/pch/pch.cc b/be/src/pch/pch.cc
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/be/src/pch/pch.h b/be/src/pch/pch.h
new file mode 100644
index 0000000000..c2af3763ce
--- /dev/null
+++ b/be/src/pch/pch.h
@@ -0,0 +1,527 @@
+// CLucene headers
+#include <CLucene.h>
+#include <CLucene/analysis/LanguageBasedAnalyzer.h>
+#include <CLucene/clucene-config.h>
+#include <CLucene/debug/error.h>
+#include <CLucene/debug/mem.h>
+#include <CLucene/index/IndexReader.h>
+#include <CLucene/search/IndexSearcher.h>
+#include <CLucene/store/Directory.h>
+#include <CLucene/store/IndexInput.h>
+#include <CLucene/store/IndexOutput.h>
+#include <CLucene/store/RAMDirectory.h>
+#include <CLucene/util/Misc.h>
+#include <CLucene/util/bkd/bkd_reader.h>
+
+// arrow headers
+#include <arrow/array/array_base.h>
+#include <arrow/array/array_decimal.h>
+#include <arrow/array/array_primitive.h>
+#include <arrow/array/builder_base.h>
+#include <arrow/array/builder_binary.h>
+#include <arrow/array/builder_decimal.h>
+#include <arrow/array/builder_nested.h>
+#include <arrow/array/builder_primitive.h>
+#include <arrow/array/data.h>
+#include <arrow/buffer.h>
+#include <arrow/io/interfaces.h>
+#include <arrow/io/memory.h>
+#include <arrow/io/type_fwd.h>
+#include <arrow/ipc/writer.h>
+#include <arrow/pretty_print.h>
+#include <arrow/record_batch.h>
+#include <arrow/result.h>
+#include <arrow/status.h>
+#include <arrow/type.h>
+#include <arrow/type_fwd.h>
+#include <arrow/util/decimal.h>
+#include <arrow/visit_type_inline.h>
+#include <arrow/visitor.h>
+
+// aws headers
+#include <aws/core/Aws.h>
+#include <aws/core/auth/AWSAuthSigner.h>
+#include <aws/core/auth/AWSCredentials.h>
+#include <aws/core/client/AWSError.h>
+#include <aws/core/client/ClientConfiguration.h>
+#include <aws/core/http/HttpResponse.h>
+#include <aws/core/utils/Array.h>
+#include <aws/core/utils/HashingUtils.h>
+#include <aws/core/utils/Outcome.h>
+#include <aws/core/utils/logging/LogLevel.h>
+#include <aws/core/utils/logging/LogSystemInterface.h>
+#include <aws/core/utils/memory/stl/AWSAllocator.h>
+#include <aws/core/utils/memory/stl/AWSMap.h>
+#include <aws/core/utils/memory/stl/AWSStreamFwd.h>
+#include <aws/core/utils/memory/stl/AWSString.h>
+#include <aws/core/utils/memory/stl/AWSStringStream.h>
+#include <aws/core/utils/memory/stl/AWSVector.h>
+#include <aws/core/utils/stream/PreallocatedStreamBuf.h>
+#include <aws/core/utils/threading/Executor.h>
+#include <aws/s3/S3Client.h>
+#include <aws/s3/S3Errors.h>
+#include <aws/s3/model/AbortMultipartUploadRequest.h>
+#include <aws/s3/model/AbortMultipartUploadResult.h>
+#include <aws/s3/model/CompleteMultipartUploadRequest.h>
+#include <aws/s3/model/CompleteMultipartUploadResult.h>
+#include <aws/s3/model/CompletedMultipartUpload.h>
+#include <aws/s3/model/CompletedPart.h>
+#include <aws/s3/model/CopyObjectRequest.h>
+#include <aws/s3/model/CopyObjectResult.h>
+#include <aws/s3/model/CreateMultipartUploadRequest.h>
+#include <aws/s3/model/CreateMultipartUploadResult.h>
+#include <aws/s3/model/Delete.h>
+#include <aws/s3/model/DeleteObjectRequest.h>
+#include <aws/s3/model/DeleteObjectResult.h>
+#include <aws/s3/model/DeleteObjectsRequest.h>
+#include <aws/s3/model/DeleteObjectsResult.h>
+#include <aws/s3/model/Error.h>
+#include <aws/s3/model/GetObjectRequest.h>
+#include <aws/s3/model/GetObjectResult.h>
+#include <aws/s3/model/HeadObjectRequest.h>
+#include <aws/s3/model/HeadObjectResult.h>
+#include <aws/s3/model/ListObjectsV2Request.h>
+#include <aws/s3/model/ListObjectsV2Result.h>
+#include <aws/s3/model/Object.h>
+#include <aws/s3/model/ObjectIdentifier.h>
+#include <aws/s3/model/PutObjectRequest.h>
+#include <aws/s3/model/PutObjectResult.h>
+#include <aws/s3/model/UploadPartRequest.h>
+#include <aws/s3/model/UploadPartResult.h>
+#include <aws/transfer/TransferHandle.h>
+#include <aws/transfer/TransferManager.h>
+
+// bitshuffle headers
+#include <bitshuffle/bitshuffle.h>
+
+// brpc headers
+#include <brpc/adaptive_connection_type.h>
+#include <brpc/adaptive_protocol_type.h>
+#include <brpc/callback.h>
+#include <brpc/channel.h>
+#include <brpc/closure_guard.h>
+#include <brpc/controller.h>
+#include <brpc/http_header.h>
+#include <brpc/http_method.h>
+#include <brpc/server.h>
+#include <brpc/ssl_options.h>
+#include <brpc/uri.h>
+
+// bthread headers
+#include <bthread/bthread.h>
+#include <bthread/condition_variable.h>
+#include <bthread/mutex.h>
+#include <bthread/types.h>
+
+// butil headers
+#include <butil/endpoint.h>
+#include <butil/errno.h>
+#include <butil/fast_rand.h>
+#include <butil/fd_utility.h>
+#include <butil/iobuf.h>
+#include <butil/iobuf_inl.h>
+#include <butil/macros.h>
+#include <butil/time.h>
+
+// bvar headers
+#include <bvar/bvar.h>
+#include <bvar/latency_recorder.h>
+#include <bvar/reducer.h>
+#include <bvar/window.h>
+
+// cctz headers
+#include <cctz/civil_time.h>
+#include <cctz/civil_time_detail.h>
+#include <cctz/time_zone.h>
+
+// event2 headers
+#include <event2/buffer.h>
+#include <event2/event.h>
+#include <event2/http.h>
+#include <event2/http_struct.h>
+#include <event2/keyvalq_struct.h>
+#include <event2/thread.h>
+
+// fast_float headers
+#include <fast_float/fast_float.h>
+#include <fast_float/parse_number.h>
+
+// fmt headers
+#include <fmt/core.h>
+#include <fmt/format.h>
+#include <fmt/ranges.h>
+
+// gen_cpp headers
+#include <gen_cpp/AgentService_types.h>
+#include <gen_cpp/BackendService.h>
+#include <gen_cpp/BackendService_types.h>
+#include <gen_cpp/DataSinks_types.h>
+#include <gen_cpp/Data_types.h>
+#include <gen_cpp/Descriptors_types.h>
+#include <gen_cpp/DorisExternalService_types.h>
+#include <gen_cpp/Exprs_types.h>
+#include <gen_cpp/FrontendService.h>
+#include <gen_cpp/FrontendService_types.h>
+#include <gen_cpp/HeartbeatService.h>
+#include <gen_cpp/HeartbeatService_constants.h>
+#include <gen_cpp/HeartbeatService_types.h>
+#include <gen_cpp/MasterService_types.h>
+#include <gen_cpp/Metrics_types.h>
+#include <gen_cpp/Opcodes_types.h>
+#include <gen_cpp/PaloBrokerService_types.h>
+#include <gen_cpp/PaloInternalService_types.h>
+#include <gen_cpp/Partitions_types.h>
+#include <gen_cpp/PlanNodes_types.h>
+#include <gen_cpp/Planner_types.h>
+#include <gen_cpp/QueryPlanExtra_types.h>
+#include <gen_cpp/RuntimeProfile_types.h>
+#include <gen_cpp/Status_types.h>
+#include <gen_cpp/TPaloBrokerService.h>
+#include <gen_cpp/Types_constants.h>
+#include <gen_cpp/Types_types.h>
+#include <gen_cpp/data.pb.h>
+#include <gen_cpp/descriptors.pb.h>
+#include <gen_cpp/function_service.pb.h>
+#include <gen_cpp/internal_service.pb.h>
+#include <gen_cpp/olap_common.pb.h>
+#include <gen_cpp/olap_file.pb.h>
+#include <gen_cpp/parquet_types.h>
+#include <gen_cpp/segment_v2.pb.h>
+#include <gen_cpp/types.pb.h>
+#include <gen_cpp/version.h>
+
+// gflags headers
+#include <gflags/gflags.h>
+#include <gflags/gflags_declare.h>
+
+// glog headers
+#include <glog/logging.h>
+#include <glog/stl_logging.h>
+
+// protobuf headers
+#include <google/protobuf/service.h>
+#include <google/protobuf/stubs/callback.h>
+#include <google/protobuf/stubs/common.h>
+#include <google/protobuf/stubs/port.h>
+
+// gperftools headers
+#include <gperftools/malloc_extension.h>
+#include <gperftools/malloc_hook.h>
+#include <gperftools/nallocx.h>
+#include <gperftools/tcmalloc.h>
+
+// hs headers
+#include <hs/hs.h>
+#include <hs/hs_common.h>
+#include <hs/hs_compile.h>
+#include <hs/hs_runtime.h>
+
+// jemalloc headers
+#include <jemalloc/jemalloc.h>
+
+// json2pb headers
+#include <json2pb/json_to_pb.h>
+#include <json2pb/pb_to_json.h>
+#include <json2pb/zero_copy_stream_reader.h>
+
+// lz4 headers
+#include <lz4/lz4.h>
+#include <lz4/lz4frame.h>
+
+// lzo headers
+#include <lzo/lzo1x.h>
+#include <lzo/lzoconf.h>
+
+// mysql headers
+#include <mysql/mysql.h>
+
+// openssl headers
+#include <openssl/err.h>
+#include <openssl/evp.h>
+#include <openssl/md5.h>
+#include <openssl/ossl_typ.h>
+
+// opentelemetry headers
+#include <opentelemetry/context/context.h>
+#include <opentelemetry/context/propagation/global_propagator.h>
+#include <opentelemetry/context/propagation/text_map_propagator.h>
+#include <opentelemetry/context/runtime_context.h>
+#include <opentelemetry/exporters/otlp/otlp_environment.h>
+#include <opentelemetry/exporters/otlp/otlp_http_exporter.h>
+#include <opentelemetry/exporters/zipkin/zipkin_exporter.h>
+#include <opentelemetry/nostd/shared_ptr.h>
+#include <opentelemetry/nostd/string_view.h>
+#include <opentelemetry/nostd/variant.h>
+#include <opentelemetry/sdk/common/attribute_utils.h>
+#include <opentelemetry/sdk/common/global_log_handler.h>
+#include <opentelemetry/sdk/resource/resource.h>
+#include <opentelemetry/sdk/resource/resource_detector.h>
+#include <opentelemetry/sdk/trace/batch_span_processor.h>
+#include <opentelemetry/sdk/trace/exporter.h>
+#include <opentelemetry/sdk/trace/processor.h>
+#include <opentelemetry/sdk/trace/recordable.h>
+#include <opentelemetry/sdk/trace/tracer_provider.h>
+#include <opentelemetry/trace/canonical_code.h>
+#include <opentelemetry/trace/context.h>
+#include <opentelemetry/trace/noop.h>
+#include <opentelemetry/trace/propagation/http_trace_context.h>
+#include <opentelemetry/trace/provider.h>
+#include <opentelemetry/trace/scope.h>
+#include <opentelemetry/trace/span.h>
+#include <opentelemetry/trace/span_context.h>
+#include <opentelemetry/trace/span_metadata.h>
+#include <opentelemetry/trace/span_startoptions.h>
+#include <opentelemetry/trace/tracer.h>
+#include <opentelemetry/trace/tracer_provider.h>
+
+// parallel_hashmap headers
+#include <parallel_hashmap/btree.h>
+#include <parallel_hashmap/phmap.h>
+
+// parquet headers
+#include <parquet/arrow/reader.h>
+#include <parquet/column_writer.h>
+#include <parquet/exception.h>
+#include <parquet/file_reader.h>
+#include <parquet/file_writer.h>
+#include <parquet/metadata.h>
+#include <parquet/platform.h>
+#include <parquet/properties.h>
+#include <parquet/schema.h>
+#include <parquet/type_fwd.h>
+#include <parquet/types.h>
+
+// rapidjson headers
+#include <rapidjson/allocators.h>
+#include <rapidjson/document.h>
+#include <rapidjson/encodings.h>
+#include <rapidjson/error/en.h>
+#include <rapidjson/prettywriter.h>
+#include <rapidjson/rapidjson.h>
+#include <rapidjson/reader.h>
+#include <rapidjson/stringbuffer.h>
+#include <rapidjson/writer.h>
+
+// re2 headers
+#include <re2/re2.h>
+#include <re2/stringpiece.h>
+
+// rocksdb headers
+#include <rocksdb/iterator.h>
+#include <rocksdb/status.h>
+
+// sanitizer headers
+#include <sanitizer/asan_interface.h>
+#include <sanitizer/lsan_interface.h>
+
+// simdjson headers
+#include <simdjson/common_defs.h>
+#include <simdjson/simdjson.h>
+
+// snappy headers
+#include <snappy/snappy-sinksource.h>
+#include <snappy/snappy.h>
+
+// thrift headers
+#include <thrift/TApplicationException.h>
+#include <thrift/TOutput.h>
+#include <thrift/Thrift.h>
+#include <thrift/concurrency/ThreadFactory.h>
+#include <thrift/concurrency/ThreadManager.h>
+#include <thrift/protocol/TBinaryProtocol.h>
+#include <thrift/protocol/TCompactProtocol.h>
+#include <thrift/protocol/TDebugProtocol.h>
+#include <thrift/protocol/TJSONProtocol.h>
+#include <thrift/protocol/TProtocol.h>
+#include <thrift/server/TNonblockingServer.h>
+#include <thrift/server/TServer.h>
+#include <thrift/server/TThreadPoolServer.h>
+#include <thrift/server/TThreadedServer.h>
+#include <thrift/transport/TBufferTransports.h>
+#include <thrift/transport/TNonblockingServerSocket.h>
+#include <thrift/transport/TServerSocket.h>
+#include <thrift/transport/TSocket.h>
+#include <thrift/transport/TTransport.h>
+#include <thrift/transport/TTransportException.h>
+
+// boost headers
+#include <boost/algorithm/string.hpp>
+#include <boost/algorithm/string/case_conv.hpp>
+#include <boost/algorithm/string/classification.hpp>
+#include <boost/algorithm/string/detail/classification.hpp>
+#include <boost/algorithm/string/join.hpp>
+#include <boost/algorithm/string/predicate.hpp>
+#include <boost/algorithm/string/predicate_facade.hpp>
+#include <boost/algorithm/string/replace.hpp>
+#include <boost/algorithm/string/split.hpp>
+#include <boost/algorithm/string/trim.hpp>
+#include <boost/container/detail/std_fwd.hpp>
+#include <boost/container/pmr/monotonic_buffer_resource.hpp>
+#include <boost/container/pmr/vector.hpp>
+#include <boost/container_hash/hash.hpp>
+#include <boost/core/noncopyable.hpp>
+#include <boost/dynamic_bitset.hpp>
+#include <boost/intrusive/detail/algo_type.hpp>
+#include <boost/intrusive/list.hpp>
+#include <boost/intrusive/list_hook.hpp>
+#include <boost/iterator/iterator_adaptor.hpp>
+#include <boost/iterator/iterator_facade.hpp>
+#include <boost/lexical_cast.hpp>
+#include <boost/lexical_cast/bad_lexical_cast.hpp>
+#include <boost/noncopyable.hpp>
+#include <boost/operators.hpp>
+#include <boost/preprocessor/repetition/repeat_from_to.hpp>
+#include <boost/random/mersenne_twister.hpp>
+#include <boost/shared_ptr.hpp>
+#include <boost/stacktrace.hpp>
+#include <boost/token_functions.hpp>
+#include <boost/tokenizer.hpp>
+#include <boost/type_index/type_index_facade.hpp>
+#include <boost/uuid/random_generator.hpp>
+#include <boost/uuid/uuid.hpp>
+#include <boost/uuid/uuid_generators.hpp>
+#include <boost/uuid/uuid_io.hpp>
+
+// roaring headers
+#include <roaring/roaring.hh>
+
+// others
+#include <arpa/inet.h>
+#include <assert.h>
+#include <bzlib.h>
+#include <ctype.h>
+#include <curl/curl.h>
+#include <curl/system.h>
+#include <cxxabi.h>
+#include <dlfcn.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <fenv.h>
+#include <float.h>
+#include <iconv.h>
+#include <inttypes.h>
+#include <jni.h>
+#include <jni_md.h>
+#include <libdivide.h>
+#include <libgen.h>
+#include <librdkafka/rdkafkacpp.h>
+#include <limits.h>
+#include <math.h>
+#include <netdb.h>
+#include <netinet/in.h>
+#include <pdqsort.h>
+#include <pthread.h>
+#include <sched.h>
+#include <setjmp.h>
+#include <signal.h>
+#include <simdjson.h>
+#include <snappy.h>
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <strings.h>
+#include <sys/mman.h>
+#include <sys/param.h>
+#include <sys/resource.h>
+#include <sys/socket.h>
+#include <sys/stat.h>
+#include <sys/syscall.h>
+#include <sys/sysctl.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <sys/ucontext.h>
+#include <sys/uio.h>
+#include <sys/un.h>
+#include <time.h>
+#include <wchar.h>
+#include <zconf.h>
+#include <zlib.h>
+#include <zstd.h>
+#include <zstd_errors.h>
+
+#include <algorithm>
+#include <array>
+#include <atomic>
+#include <bitset>
+#include <cassert>
+#include <cctype>
+#include <cerrno>
+#include <cfloat>
+#include <charconv>
+#include <chrono>
+#include <cinttypes>
+#include <climits>
+#include <cmath>
+#include <complex>
+#include <condition_variable>
+#include <csignal>
+#include <cstdarg>
+#include <cstddef>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <ctime>
+#include <deque>
+#include <exception>
+#include <filesystem>
+#include <fstream>
+#include <functional>
+#include <future>
+#include <initializer_list>
+#include <iomanip>
+#include <iosfwd>
+#include <iostream>
+#include <istream>
+#include <iterator>
+#include <limits>
+#include <list>
+#include <map>
+#include <memory>
+#include <mutex>
+#include <new>
+#include <numeric>
+#include <optional>
+#include <ostream>
+#include <queue>
+#include <random>
+#include <ratio>
+#include <regex>
+#include <set>
+#include <shared_mutex>
+#include <sstream>
+#include <stack>
+#include <stdexcept>
+#include <string>
+#include <string_view>
+#include <system_error>
+#include <thread>
+#include <tuple>
+#include <type_traits>
+#include <typeindex>
+#include <typeinfo>
+#include <unordered_map>
+#include <unordered_set>
+#include <utility>
+#include <variant>
+#include <vector>
+
+#ifndef __APPLE__
+#include <byteswap.h>
+#include <endian.h>
+#include <features.h>
+#include <immintrin.h>
+#include <linux/perf_event.h>
+#include <malloc.h>
+#include <mmintrin.h>
+#include <sys/prctl.h>
+#include <sys/sysinfo.h>
+#include <ucontext.h>
+#include <unistd.h>
+
+#include <ext/pb_ds/priority_queue.hpp>
+#include <memory_resource>
+#endif
\ No newline at end of file
diff --git a/be/src/pipeline/CMakeLists.txt b/be/src/pipeline/CMakeLists.txt
index 63905ec36b..03d3410e7b 100644
--- a/be/src/pipeline/CMakeLists.txt
+++ b/be/src/pipeline/CMakeLists.txt
@@ -68,4 +68,6 @@ endif ()
add_library(Pipeline STATIC
${PIPELINE_FILES}
- )
+)
+
+pch_reuse(Pipeline)
diff --git a/be/src/runtime/CMakeLists.txt b/be/src/runtime/CMakeLists.txt
index d596c610ad..44054bce73 100644
--- a/be/src/runtime/CMakeLists.txt
+++ b/be/src/runtime/CMakeLists.txt
@@ -88,4 +88,6 @@ endif()
add_library(Runtime STATIC
${RUNTIME_FILES}
- )
+)
+
+pch_reuse(Runtime)
diff --git a/be/src/service/CMakeLists.txt b/be/src/service/CMakeLists.txt
index f495eb4841..9446fdf700 100644
--- a/be/src/service/CMakeLists.txt
+++ b/be/src/service/CMakeLists.txt
@@ -30,11 +30,15 @@ add_library(Service
internal_service.cpp
)
+pch_reuse(Service)
+
if (${MAKE_TEST} STREQUAL "OFF")
add_executable(doris_be
doris_main.cpp
)
+ pch_reuse(doris_be)
+
# This permits libraries loaded by dlopen to link to the symbols in the
program.
set_target_properties(doris_be PROPERTIES ENABLE_EXPORTS 1)
diff --git a/be/src/tools/CMakeLists.txt b/be/src/tools/CMakeLists.txt
index 1f583e01a4..bc73713b2f 100644
--- a/be/src/tools/CMakeLists.txt
+++ b/be/src/tools/CMakeLists.txt
@@ -25,6 +25,8 @@ add_executable(meta_tool
meta_tool.cpp
)
+pch_reuse(meta_tool)
+
# This permits libraries loaded by dlopen to link to the symbols in the
program.
set_target_properties(meta_tool PROPERTIES ENABLE_EXPORTS 1)
diff --git a/be/src/util/CMakeLists.txt b/be/src/util/CMakeLists.txt
index 8c217f0acd..f2804018be 100644
--- a/be/src/util/CMakeLists.txt
+++ b/be/src/util/CMakeLists.txt
@@ -21,6 +21,8 @@ set(LIBRARY_OUTPUT_PATH "${BUILD_DIR}/src/util")
# where to put generated binaries
set(EXECUTABLE_OUTPUT_PATH "${BUILD_DIR}/src/util")
+add_library(cityhash cityhash102/city.cc)
+
set(UTIL_FILES
arrow/row_batch.cpp
arrow/utils.cpp
@@ -49,7 +51,6 @@ set(UTIL_FILES
perf_counters.cpp
runtime_profile.cpp
static_asserts.cpp
- string_parser.cpp
thrift_util.cpp
thrift_client.cpp
thrift_server.cpp
@@ -91,7 +92,6 @@ set(UTIL_FILES
s3_util.cpp
hdfs_util.cpp
time_lut.cpp
- cityhash102/city.cc
telemetry/telemetry.cpp
telemetry/brpc_carrier.cpp
telemetry/open_telemetry_scop_wrapper.hpp
@@ -101,6 +101,7 @@ set(UTIL_FILES
jni_native_method.cpp
)
+
if (OS_MACOSX)
list(REMOVE_ITEM UTIL_FILES perf_counters.cpp disk_info.cpp)
list(APPEND UTIL_FILES perf_counters_mac.cpp disk_info_mac.cpp)
@@ -109,4 +110,6 @@ endif()
add_library(Util STATIC
${UTIL_FILES}
)
+target_link_libraries(Util cityhash)
+pch_reuse(Util)
\ No newline at end of file
diff --git a/be/src/util/string_parser.cpp b/be/src/util/string_parser.cpp
deleted file mode 100644
index 16e24e6154..0000000000
--- a/be/src/util/string_parser.cpp
+++ /dev/null
@@ -1,48 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-// This file is copied from
-//
https://github.com/apache/impala/blob/branch-2.9.0/be/src/util/string-parser.cc
-// and modified by Doris
-
-#include "string_parser.hpp"
-
-#include "runtime/large_int_value.h"
-#include "vec/common/int_exp.h"
-
-namespace doris {
-
-template <>
-__int128 StringParser::numeric_limits<__int128>(bool negative) {
- return negative ? MIN_INT128 : MAX_INT128;
-}
-
-template <>
-int32_t StringParser::get_scale_multiplier(int scale) {
- return common::exp10_i32(scale);
-}
-
-template <>
-int64_t StringParser::get_scale_multiplier(int scale) {
- return common::exp10_i64(scale);
-}
-
-template <>
-__int128 StringParser::get_scale_multiplier(int scale) {
- return common::exp10_i128(scale);
-}
-
-} // namespace doris
diff --git a/be/src/util/string_parser.hpp b/be/src/util/string_parser.hpp
index 2910cad9c9..c376227a46 100644
--- a/be/src/util/string_parser.hpp
+++ b/be/src/util/string_parser.hpp
@@ -38,6 +38,8 @@
// IWYU pragma: no_include <opentelemetry/common/threadlocal.h>
#include "common/compiler_util.h" // IWYU pragma: keep
#include "common/status.h"
+#include "runtime/large_int_value.h"
+#include "vec/common/int_exp.h"
#include "vec/data_types/data_type_decimal.h"
namespace doris {
@@ -78,10 +80,27 @@ public:
};
template <typename T>
- static T numeric_limits(bool negative);
+ static T numeric_limits(bool negative) {
+ if constexpr (std::is_same_v<T, __int128>) {
+ return negative ? MIN_INT128 : MAX_INT128;
+ } else {
+ return negative ? std::numeric_limits<T>::min() :
std::numeric_limits<T>::max();
+ }
+ }
template <typename T>
- static T get_scale_multiplier(int scale);
+ static T get_scale_multiplier(int scale) {
+ static_assert(std::is_same_v<T, int32_t> || std::is_same_v<T, int64_t>
||
+ std::is_same_v<T, __int128>,
+ "You can only instantiate as int32_t, int64_t,
__int128.");
+ if constexpr (std::is_same_v<T, int32_t>) {
+ return common::exp10_i32(scale);
+ } else if constexpr (std::is_same_v<T, int64_t>) {
+ return common::exp10_i64(scale);
+ } else if constexpr (std::is_same_v<T, __int128>) {
+ return common::exp10_i128(scale);
+ }
+ }
// This is considerably faster than glibc's implementation (25x).
// In the case of overflow, the max/min value for the data type will be
returned.
@@ -504,14 +523,6 @@ inline bool StringParser::string_to_bool_internal(const
char* s, int len, ParseR
return false;
}
-template <>
-__int128 StringParser::numeric_limits<__int128>(bool negative);
-
-template <typename T>
-T StringParser::numeric_limits(bool negative) {
- return negative ? std::numeric_limits<T>::min() :
std::numeric_limits<T>::max();
-}
-
template <>
inline int StringParser::StringParseTraits<uint8_t>::max_ascii_len() {
return 3;
diff --git a/be/src/util/thrift_util.cpp b/be/src/util/thrift_util.cpp
index af0a206d4e..9356ba8bed 100644
--- a/be/src/util/thrift_util.cpp
+++ b/be/src/util/thrift_util.cpp
@@ -44,8 +44,14 @@ class TProtocol;
#ifdef UNLIKELY
#undef UNLIKELY
#endif
+#ifndef SIGNED_RIGHT_SHIFT_IS
#define SIGNED_RIGHT_SHIFT_IS 1
+#endif
+
+#ifndef ARITHMETIC_RIGHT_SHIFT
#define ARITHMETIC_RIGHT_SHIFT 1
+#endif
+
#include <thrift/protocol/TCompactProtocol.h>
#include <sstream>
diff --git a/be/src/util/trace_metrics.cpp b/be/src/util/trace_metrics.cpp
index 4f053c1d36..4b876847bd 100644
--- a/be/src/util/trace_metrics.cpp
+++ b/be/src/util/trace_metrics.cpp
@@ -35,9 +35,6 @@ using std::string;
namespace doris {
-// Make glog's STL-compatible operators visible inside this namespace.
-using ::operator<<;
-
namespace {
static SpinLock g_intern_map_lock;
diff --git a/be/src/vec/CMakeLists.txt b/be/src/vec/CMakeLists.txt
index 086a301456..98375ab8e0 100644
--- a/be/src/vec/CMakeLists.txt
+++ b/be/src/vec/CMakeLists.txt
@@ -359,4 +359,6 @@ endif ()
add_library(Vec STATIC
${VEC_FILES}
- )
+)
+
+pch_reuse(Vec)
diff --git a/be/src/vec/olap/block_reader.cpp b/be/src/vec/olap/block_reader.cpp
index d8ce3d8a84..5ae2ae5e3e 100644
--- a/be/src/vec/olap/block_reader.cpp
+++ b/be/src/vec/olap/block_reader.cpp
@@ -110,7 +110,7 @@ Status BlockReader::_init_collect_iter(const ReaderParams&
read_params) {
auto& rs_reader = read_params.rs_readers[i];
// _vcollect_iter.topn_next() will init rs_reader by itself
if (!_vcollect_iter.use_topn_next()) {
- RETURN_NOT_OK(rs_reader->init(
+ RETURN_IF_ERROR(rs_reader->init(
&_reader_context,
is_empty ? std::pair {0, 0} :
read_params.rs_readers_segment_offsets[i]));
}
@@ -173,7 +173,7 @@ void BlockReader::_init_agg_state(const ReaderParams&
read_params) {
}
Status BlockReader::init(const ReaderParams& read_params) {
- RETURN_NOT_OK(TabletReader::init(read_params));
+ RETURN_IF_ERROR(TabletReader::init(read_params));
int32_t return_column_size = read_params.origin_return_columns->size();
_return_columns_loc.resize(read_params.return_columns.size());
diff --git a/be/src/vec/olap/vcollect_iterator.cpp
b/be/src/vec/olap/vcollect_iterator.cpp
index 1045621c9c..2af1a9ac0d 100644
--- a/be/src/vec/olap/vcollect_iterator.cpp
+++ b/be/src/vec/olap/vcollect_iterator.cpp
@@ -279,7 +279,7 @@ Status VCollectIterator::_topn_next(Block* block) {
for (auto rs_reader : _rs_readers) {
// init will prune segment by _reader_context.conditions and
_reader_context.runtime_conditions
- RETURN_NOT_OK(rs_reader->init(&_reader->_reader_context));
+ RETURN_IF_ERROR(rs_reader->init(&_reader->_reader_context));
// read _topn_limit rows from this rs
size_t read_rows = 0;
@@ -481,7 +481,7 @@ Status
VCollectIterator::Level0Iterator::_refresh_current_row() {
}
if (UNLIKELY(_reader->_reader_context.record_rowids)) {
-
RETURN_NOT_OK(_rs_reader->current_block_row_locations(&_block_row_locations));
+
RETURN_IF_ERROR(_rs_reader->current_block_row_locations(&_block_row_locations));
}
}
} while (!_is_empty());
@@ -497,7 +497,7 @@ Status
VCollectIterator::Level0Iterator::next(IteratorRowRef* ref) {
_ref.row_pos++;
}
- RETURN_NOT_OK(_refresh_current_row());
+ RETURN_IF_ERROR(_refresh_current_row());
if (_get_data_by_ref) {
_ref = _block_view[_current];
@@ -522,7 +522,7 @@ Status VCollectIterator::Level0Iterator::next(Block* block)
{
return Status::Error<END_OF_FILE>();
}
if (UNLIKELY(_reader->_reader_context.record_rowids)) {
-
RETURN_NOT_OK(_rs_reader->current_block_row_locations(&_block_row_locations));
+
RETURN_IF_ERROR(_rs_reader->current_block_row_locations(&_block_row_locations));
}
return Status::OK();
}
diff --git a/be/src/vec/olap/vertical_block_reader.cpp
b/be/src/vec/olap/vertical_block_reader.cpp
index d09299857f..d9f4fb3256 100644
--- a/be/src/vec/olap/vertical_block_reader.cpp
+++ b/be/src/vec/olap/vertical_block_reader.cpp
@@ -67,7 +67,7 @@ Status VerticalBlockReader::_get_segment_iterators(const
ReaderParams& read_para
// segment iterator will be inited here
// In vertical compaction, every group will load segment so we should
cache
// segment to avoid tot many s3 head request
- RETURN_NOT_OK(
+ RETURN_IF_ERROR(
rs_reader->get_segment_iterators(&_reader_context,
segment_iters, {0, 0}, true));
// if segments overlapping, all segment iterator should be inited in
// heap merge iterator. If segments are none overlapping, only first
segment of this
@@ -187,7 +187,7 @@ void VerticalBlockReader::_init_agg_state(const
ReaderParams& read_params) {
Status VerticalBlockReader::init(const ReaderParams& read_params) {
StorageReadOptions opts;
_reader_context.batch_size = opts.block_row_max;
- RETURN_NOT_OK(TabletReader::init(read_params));
+ RETURN_IF_ERROR(TabletReader::init(read_params));
auto status = _init_collect_iter(read_params);
if (!status.ok()) {
diff --git a/build.sh b/build.sh
index bfbba9b041..e1ef0a9ae4 100755
--- a/build.sh
+++ b/build.sh
@@ -380,6 +380,7 @@ echo "Get params:
ENABLE_STACKTRACE -- ${ENABLE_STACKTRACE}
DENABLE_CLANG_COVERAGE -- ${DENABLE_CLANG_COVERAGE}
DISPLAY_BUILD_TIME -- ${DISPLAY_BUILD_TIME}
+ ENABLE_PCH -- ${ENABLE_PCH}
"
# Clean and build generated code
@@ -445,6 +446,7 @@ if [[ "${BUILD_BE}" -eq 1 ]]; then
-DSTRIP_DEBUG_INFO="${STRIP_DEBUG_INFO}" \
-DUSE_DWARF="${USE_DWARF}" \
-DDISPLAY_BUILD_TIME="${DISPLAY_BUILD_TIME}" \
+ -DENABLE_PCH="${ENABLE_PCH}" \
-DUSE_MEM_TRACKER="${USE_MEM_TRACKER}" \
-DUSE_JEMALLOC="${USE_JEMALLOC}" \
-DUSE_BTHREAD_SCANNER="${USE_BTHREAD_SCANNER}" \
@@ -453,6 +455,7 @@ if [[ "${BUILD_BE}" -eq 1 ]]; then
-DGLIBC_COMPATIBILITY="${GLIBC_COMPATIBILITY}" \
-DEXTRA_CXX_FLAGS="${EXTRA_CXX_FLAGS}" \
-DENABLE_CLANG_COVERAGE="${DENABLE_CLANG_COVERAGE}" \
+ -DDORIS_JAVA_HOME="${JAVA_HOME}" \
"${DORIS_HOME}/be"
if [[ "${OUTPUT_BE_BINARY}" -eq 1 ]]; then
diff --git a/docs/en/docs/install/source-install/compilation-general.md
b/docs/en/docs/install/source-install/compilation-general.md
index 5d01f20da2..b781f58f76 100644
--- a/docs/en/docs/install/source-install/compilation-general.md
+++ b/docs/en/docs/install/source-install/compilation-general.md
@@ -235,6 +235,23 @@ You can compile Doris directly in your own Linux
environment.
You can fix this by increasing the memory allocation for the image, 4 GB ~
8 GB, for example.
+4. When using Clang to compile Doris, PCH files will be used by default to
speed up the compilation process. The default configuration of ccache may cause
PCH files to be unable to be cached, or the cache to be unable to be hit,
resulting in PCH being repeatedly compiled, slowing down the compilation speed.
The following configuration is required:
+
+ To make ccache cache PCH files:
+ ```shell
+ export CCACHE_PCH_EXTSUM=true
+ ccache --set-config=sloppiness=pch_defines,time_macros
--set-config=pch_external_checksum=true
+ ```
+ To stop ccache from caching PCH files:
+ ```shell
+ export CCACHE_NOPCH_EXTSUM=true
+ ccache --set-config=sloppiness=default
--set-config=pch_external_checksum=false
+ ```
+ To use Clang to compile, but do not want to use PCH files to speed up the
compilation process, you need to add the parameter `ENABLE_PCH=0`
+ ```shell
+ DORIS_TOOLCHAIN=clang ENABLE_PCH=0 sh build.sh
+ ```
+
## Special Statement
Starting from version 0.13, the dependency on the two third-party libraries
[1] and [2] will be removed in the default compiled output. These two
third-party libraries are under [GNU General Public License
V3](https://www.gnu.org/licenses/gpl-3.0.en.html). This license is incompatible
with [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0), so it
will not appear in the Apache release by default.
diff --git a/docs/zh-CN/docs/install/source-install/compilation-general.md
b/docs/zh-CN/docs/install/source-install/compilation-general.md
index 282e63c33e..4f1da309d1 100644
--- a/docs/zh-CN/docs/install/source-install/compilation-general.md
+++ b/docs/zh-CN/docs/install/source-install/compilation-general.md
@@ -228,6 +228,23 @@ under the License.
尝试适当调大镜像的分配内存，推荐 4GB ~ 8GB。
+4.
在使用Clang编译Doris时会默认使用PCH文件来加速编译过程，ccache的默认配置可能会导致PCH文件无法被缓存，或者缓存无法被命中，进而导致PCH被重复编译，拖慢编译速度，需要进行如下配置：
+
+ 如需让ccache缓存PCH文件：
+ ```shell
+ export CCACHE_PCH_EXTSUM=true
+ ccache --set-config=sloppiness=pch_defines,time_macros
--set-config=pch_external_checksum=true
+ ```
+ 不再让ccache缓存PCH文件：
+ ```shell
+ export CCACHE_NOPCH_EXTSUM=true
+ ccache --set-config=sloppiness=default
--set-config=pch_external_checksum=false
+ ```
+ 使用Clang编译，但不想使用PCH文件来加速编译过程，则需要加上参数`ENABLE_PCH=0`
+ ```shell
+ DORIS_TOOLCHAIN=clang ENABLE_PCH=0 sh build.sh
+ ```
+
## 特别声明
自 0.13 版本开始，默认的编译产出中将取消对 [1] 和 [2] 两个第三方库的依赖。这两个第三方库为 [GNU General Public
License V3](https://www.gnu.org/licenses/gpl-3.0.en.html) 协议。该协议与 [Apache
License 2.0](https://www.apache.org/licenses/LICENSE-2.0) 协议不兼容，因此默认不出现在 Apache
发布版本中。
diff --git a/env.sh b/env.sh
index 75a8203e40..a928b06f16 100755
--- a/env.sh
+++ b/env.sh
@@ -132,6 +132,7 @@ if [[ "${DORIS_TOOLCHAIN}" == "gcc" ]]; then
if test -x "${DORIS_GCC_HOME}/bin/ld"; then
export DORIS_BIN_UTILS="${DORIS_GCC_HOME}/bin/"
fi
+ ENABLE_PCH='OFF'
elif [[ "${DORIS_TOOLCHAIN}" == "clang" ]]; then
# set CLANG HOME
if [[ -z "${DORIS_CLANG_HOME}" ]]; then
@@ -153,6 +154,11 @@ elif [[ "${DORIS_TOOLCHAIN}" == "clang" ]]; then
if [[ -f "${DORIS_CLANG_HOME}/bin/llvm-symbolizer" ]]; then
export ASAN_SYMBOLIZER_PATH="${DORIS_CLANG_HOME}/bin/llvm-symbolizer"
fi
+ if [[ -z "${ENABLE_PCH}" ]]; then
+ ENABLE_PCH='ON'
+ else
+ ENABLE_PCH='OFF'
+ fi
else
echo "Error: unknown DORIS_TOOLCHAIN=${DORIS_TOOLCHAIN}, currently only
'gcc' and 'clang' are supported"
exit 1
diff --git a/run-be-ut.sh b/run-be-ut.sh
index a80aa2e470..06f7d4c0c6 100755
--- a/run-be-ut.sh
+++ b/run-be-ut.sh
@@ -129,6 +129,7 @@ CMAKE_BUILD_TYPE="$(echo "${CMAKE_BUILD_TYPE}" | awk '{
print(toupper($0)) }')"
echo "Get params:
PARALLEL -- ${PARALLEL}
CLEAN -- ${CLEAN}
+ ENABLE_PCH -- ${ENABLE_PCH}
"
echo "Build Backend UT"
@@ -199,6 +200,8 @@ cd "${CMAKE_BUILD_DIR}"
-DEXTRA_CXX_FLAGS="${EXTRA_CXX_FLAGS}" \
-DENABLE_CLANG_COVERAGE="${DENABLE_CLANG_COVERAGE}" \
${CMAKE_USE_CCACHE:+${CMAKE_USE_CCACHE}} \
+ -DENABLE_PCH="${ENABLE_PCH}" \
+ -DDORIS_JAVA_HOME="${JAVA_HOME}" \
"${DORIS_HOME}/be"
"${BUILD_SYSTEM}" -j "${PARALLEL}"
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
