This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new a34d4b55f9 [dependency] Add opentelemetry related third-party library.
(#9990)
a34d4b55f9 is described below
commit a34d4b55f9872e1dd7c4dd1286f334787df9fac4
Author: luozenglin <37725793+luozeng...@users.noreply.github.com>
AuthorDate: Fri Jun 10 09:59:07 2022 +0800
[dependency] Add opentelemetry related third-party library. (#9990)
---
dist/LICENSE-dist.txt | 3 +
thirdparty/CHANGELOG.md | 5 +
thirdparty/build-thirdparty.sh | 23 ++
thirdparty/download-thirdparty.sh | 16 +
thirdparty/patches/opentelemetry-cpp-1.4.0.patch | 374 +++++++++++++++++++++++
thirdparty/vars.sh | 21 ++
6 files changed, 442 insertions(+)
diff --git a/dist/LICENSE-dist.txt b/dist/LICENSE-dist.txt
index 2330685ebb..2d9bfe9016 100644
--- a/dist/LICENSE-dist.txt
+++ b/dist/LICENSE-dist.txt
@@ -1525,12 +1525,15 @@ The Apache Software License, Version 2.0
* benchmark: 1.5.6
* simdjson: 1.0.2
* libhdfs3: commit 5fccd36
+ * opentelemetry-proto: 0.18.0
+ * opentelemetry-cpp: 1.4.0
The MIT License -- licenses/LICENSE-MIT.txt
* datatables: 1.10.25
* bootstrap: 1.17.1
* libxml2: 2.9.10
* krb5: 1.19
+ * nlohmann/json 3.10.1
LGPL -- licenes/LICENSE-LGPL.txt
* gsasl: 1.10.0/1.8.0
diff --git a/thirdparty/CHANGELOG.md b/thirdparty/CHANGELOG.md
index 9287a2838b..578eb7ffc0 100644
--- a/thirdparty/CHANGELOG.md
+++ b/thirdparty/CHANGELOG.md
@@ -2,6 +2,11 @@
This file contains version of the third-party dependency libraries in the
build-env image. The docker build-env image is apache/incubator-doris, and the
tag is `build-env-${version}`
+## v20220607
+- Added: opentelemetry-cpp 1.4.0, it was introduced for tracing.
+- Added: opentelemetry-proto 0.18.0, it is depended on by opentelemetry-cpp.
+- Added: nlohmann/json 3.10.1, it is depended on by opentelemetry-cpp.
+
## v20220606
- Added: hyperscan 5.4.0, and a patch for compilation
- Added: ragel 6.1.0, it is used by hyperscan to generate files before
compilation
diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh
index 1690e7e8f6..dfd3f0012b 100755
--- a/thirdparty/build-thirdparty.sh
+++ b/thirdparty/build-thirdparty.sh
@@ -1015,6 +1015,27 @@ build_simdjson() {
cp -r $TP_SOURCE_DIR/$SIMDJSON_SOURCE/include/* $TP_INCLUDE_DIR/
}
+# nlohmann_json
+build_nlohmann_json() {
+ check_if_source_exist $NLOHMANN_JSON_SOURCE
+ cd $TP_SOURCE_DIR/$NLOHMANN_JSON_SOURCE
+ mkdir -p $BUILD_DIR && cd $BUILD_DIR
+
+ $CMAKE_CMD -G "${GENERATOR}" -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR
-DCMAKE_PREFIX_PATH=$TP_INSTALL_DIR -DJSON_BuildTests=OFF ..
+ ${BUILD_SYSTEM} -j $PARALLEL && ${BUILD_SYSTEM} install
+}
+
+# opentelemetry
+build_opentelemetry() {
+ check_if_source_exist $OPENTELEMETRY_SOURCE
+ cd $TP_SOURCE_DIR/$OPENTELEMETRY_SOURCE
+ mkdir -p $BUILD_DIR && cd $BUILD_DIR
+
+ $CMAKE_CMD -G "${GENERATOR}" -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR
-DCMAKE_PREFIX_PATH=$TP_INSTALL_DIR -DBUILD_TESTING=OFF \
+ -DWITH_OTLP=ON -DWITH_OTLP_GRPC=OFF -DWITH_OTLP_HTTP=ON -DWITH_ZIPKIN=ON
-DWITH_EXAMPLES=OFF ..
+ ${BUILD_SYSTEM} -j $PARALLEL && ${BUILD_SYSTEM} install
+}
+
build_libunixodbc
build_openssl
build_libevent
@@ -1065,6 +1086,8 @@ build_hdfs3
build_hdfs3_with_kerberos
build_benchmark
build_simdjson
+build_nlohmann_json
+build_opentelemetry
build_libbacktrace
echo "Finished to build all thirdparties"
diff --git a/thirdparty/download-thirdparty.sh
b/thirdparty/download-thirdparty.sh
index 526e549512..30ee5e7e55 100755
--- a/thirdparty/download-thirdparty.sh
+++ b/thirdparty/download-thirdparty.sh
@@ -301,6 +301,22 @@ if [ $ROCKSDB_SOURCE == "rocksdb-5.14.2" ]; then
fi
echo "Finished patching $ROCKSDB_SOURCE"
+# opentelemetry patch is used to solve the problem that threadlocal depends on
GLIBC_2.18
+# see: https://github.com/apache/incubator-doris/pull/7911
+if [ $OPENTELEMETRY_SOURCE == "opentelemetry-cpp-1.4.0" ]; then
+ rm -rf
$TP_SOURCE_DIR/$OPENTELEMETRY_SOURCE/third_party/opentelemetry-proto/*
+ cp -r $TP_SOURCE_DIR/$OPENTELEMETRY_PROTO_SOURCE/*
$TP_SOURCE_DIR/$OPENTELEMETRY_SOURCE/third_party/opentelemetry-proto
+ mkdir -p
$TP_SOURCE_DIR/$OPENTELEMETRY_SOURCE/third_party/opentelemetry-proto/.git
+
+ cd $TP_SOURCE_DIR/$OPENTELEMETRY_SOURCE
+ if [ ! -f $PATCHED_MARK ]; then
+ patch -p1 < $TP_PATCH_DIR/opentelemetry-cpp-1.4.0.patch
+ touch $PATCHED_MARK
+ fi
+ cd -
+fi
+echo "Finished patching $OPENTELEMETRY_SOURCE"
+
# patch librdkafka to avoid crash
if [ $LIBRDKAFKA_SOURCE = "librdkafka-1.8.2" ]; then
cd $TP_SOURCE_DIR/$LIBRDKAFKA_SOURCE
diff --git a/thirdparty/patches/opentelemetry-cpp-1.4.0.patch
b/thirdparty/patches/opentelemetry-cpp-1.4.0.patch
new file mode 100644
index 0000000000..2862d11c29
--- /dev/null
+++ b/thirdparty/patches/opentelemetry-cpp-1.4.0.patch
@@ -0,0 +1,374 @@
+diff --git a/api/include/opentelemetry/common/threadlocal.h
b/api/include/opentelemetry/common/threadlocal.h
+new file mode 100644
+index 0000000..799ec6c
+--- /dev/null
++++ b/api/include/opentelemetry/common/threadlocal.h
+@@ -0,0 +1,206 @@
++// Copyright The OpenTelemetry Authors
++// SPDX-License-Identifier: Apache-2.0
++
++#pragma once
++
++#include <pthread.h>
++
++#include <memory>
++#include <mutex>
++#include <ostream>
++
++//
++// GCC can be told that a certain branch is not likely to be taken (for
++// instance, a CHECK failure), and use that information in static analysis.
++// Giving it this information can help it optimize for the common case in
++// the absence of better information (ie. -fprofile-arcs).
++//
++#ifndef PREDICT_FALSE
++#if defined(__GNUC__)
++#define PREDICT_FALSE(x) (__builtin_expect(x, 0))
++#else
++#define PREDICT_FALSE(x) x
++#endif
++#endif
++#ifndef PREDICT_TRUE
++#if defined(__GNUC__)
++#define PREDICT_TRUE(x) (__builtin_expect(!!(x), 1))
++#else
++#define PREDICT_TRUE(x) x
++#endif
++#endif
++
++// Block-scoped static thread local implementation.
++//
++// Usage is similar to a C++11 thread_local. The
BLOCK_STATIC_THREAD_LOCAL_TELEMTRY macro
++// defines a thread-local pointer to the specified type, which is lazily
++// instantiated by any thread entering the block for the first time. The
++// constructor for the type T is invoked at macro execution time, as expected,
++// and its destructor is invoked when the corresponding thread's Runnable
++// returns, or when the thread exits.
++//
++// Inspired by Poco <http://pocoproject.org/docs/Poco.ThreadLocal.html>,
++// Andrew Tomazos <http://stackoverflow.com/questions/12049684/>, and
++// the C++11 thread_local API.
++//
++// Example usage:
++//
++// // Invokes a 3-arg constructor on SomeClass:
++// BLOCK_STATIC_THREAD_LOCAL_TELEMTRY(SomeClass, instance, arg1, arg2, arg3);
++// instance->DoSomething();
++//
++#define BLOCK_STATIC_THREAD_LOCAL_TELEMTRY(T, t, ...)
\
++static __thread T* t;
\
++do {
\
++ if (PREDICT_FALSE(t == NULL)) {
\
++ t = new T(__VA_ARGS__);
\
++
internal_threadlocal::ThreadLocal::AddDestructor(internal_threadlocal::ThreadLocal::Destroy<T>,
t); \
++ }
\
++} while (false)
++
++// Class-scoped static thread local implementation.
++//
++// Very similar in implementation to the above block-scoped version, but
++// requires a bit more syntax and vigilance to use properly.
++//
++// DECLARE_STATIC_THREAD_LOCAL_TELEMETRY(Type, instance_var_) must be placed
in the
++// class header, as usual for variable declarations.
++//
++// Because these variables are static, they must also be defined in the impl
++// file with DEFINE_STATIC_THREAD_LOCAL_TELEMETRY(Type, Classname,
instance_var_),
++// which is very much like defining any static member, i.e. int Foo::member_.
++//
++// Finally, each thread must initialize the instance before using it by
calling
++// INIT_STATIC_THREAD_LOCAL_TELEMETRY(Type, instance_var_, ...). This is a
cheap
++// call, and may be invoked at the top of any method which may reference a
++// thread-local variable.
++//
++// Due to all of these requirements, you should probably declare TLS members
++// as private.
++//
++// Example usage:
++//
++// // foo.h
++// #include "kudu/utils/file.h"
++// class Foo {
++// public:
++// void DoSomething(std::string s);
++// private:
++// DECLARE_STATIC_THREAD_LOCAL_TELEMETRY(utils::File, file_);
++// };
++//
++// // foo.cc
++// #include "kudu/foo.h"
++// DEFINE_STATIC_THREAD_LOCAL_TELEMETRY(utils::File, Foo, file_);
++// void Foo::WriteToFile(std::string s) {
++// // Call constructor if necessary.
++// INIT_STATIC_THREAD_LOCAL_TELEMETRY(utils::File, file_,
"/tmp/file_location.txt");
++// file_->Write(s);
++// }
++
++// Goes in the class declaration (usually in a header file).
++// dtor must be destructed _after_ t, so it gets defined first.
++// Uses a mangled variable name for dtor since it must also be a member of the
++// class.
++#define DECLARE_STATIC_THREAD_LOCAL_TELEMETRY(T, t)
\
++static __thread T* t
++
++// You must also define the instance in the .cc file.
++#define DEFINE_STATIC_THREAD_LOCAL_TELEMETRY(T, Class, t)
\
++__thread T* Class::t
++
++// Must be invoked at least once by each thread that will access t.
++#define INIT_STATIC_THREAD_LOCAL_TELEMETRY(T, t, ...)
\
++do {
\
++ if (PREDICT_FALSE(t == NULL)) {
\
++ t = new T(__VA_ARGS__);
\
++
internal_threadlocal::ThreadLocal::AddDestructor(internal_threadlocal::ThreadLocal::Destroy<T>,
t); \
++ }
\
++} while (false)
++
++// Internal implementation below.
++OPENTELEMETRY_BEGIN_NAMESPACE
++namespace internal_threadlocal
++{
++
++// One key used by the entire process to attach destructors on thread exit.
++static pthread_key_t destructors_key;
++
++static std::once_flag once_init;
++
++namespace
++{
++// List of destructors for all thread locals instantiated on a given thread.
++struct PerThreadDestructorList
++{
++ void (*destructor)(void *);
++ void *arg;
++ PerThreadDestructorList *next;
++};
++
++} // anonymous namespace
++
++class ThreadLocal {
++public:
++// Destroy the passed object of type T.
++ template <class T>
++ static void Destroy(void *t)
++ {
++ // With tcmalloc, this should be pretty cheap (same thread as new).
++ delete reinterpret_cast<T *>(t);
++ }
++
++ // Call all the destructors associated with all THREAD_LOCAL instances in
this
++// thread.
++ static void InvokeDestructors(void *t)
++ {
++ PerThreadDestructorList *d = reinterpret_cast<PerThreadDestructorList
*>(t);
++ while (d != nullptr)
++ {
++ d->destructor(d->arg);
++ PerThreadDestructorList *next = d->next;
++ delete d;
++ d = next;
++ }
++ }
++
++// This key must be initialized only once.
++ static void CreateKey()
++ {
++ int ret = pthread_key_create(&destructors_key, &InvokeDestructors);
++ // Linux supports up to 1024 keys, we will use only one for all thread
locals.
++/* if (ret != 0)
++ {
++ std::stringstream ss;
++ ss << "[thread local] pthread_key_create() failed, cannot add
destructor to thread: "
++ << "error " << ret;
++ OTEL_INTERNAL_LOG_ERROR(ss.str());
++ }*/
++ }
++
++// Adds a destructor to the list.
++ static void AddDestructor(void (*destructor)(void *), void *arg)
++ {
++ std::call_once(once_init, &CreateKey);
++
++ // Returns NULL if nothing is set yet.
++ std::unique_ptr<PerThreadDestructorList> p(new PerThreadDestructorList());
++ p->destructor = destructor;
++ p->arg = arg;
++ p->next = reinterpret_cast<PerThreadDestructorList
*>(pthread_getspecific(destructors_key));
++ int ret = pthread_setspecific(destructors_key, p.release());
++ // The only time this check should fail is if we are out of memory, or if
++ // somehow key creation failed, which should be caught by the above CHECK.
++/* if (ret != 0)
++ {
++ std::stringstream ss;
++ ss << "[thread local] pthread_setspecific() failed, cannot update
destructor list: "
++ << "error " << ret;
++ OTEL_INTERNAL_LOG_ERROR(ss.str());
++ }*/
++ }
++
++};
++
++} // namespace threadlocal
++OPENTELEMETRY_END_NAMESPACE
+\ No newline at end of file
+diff --git a/api/include/opentelemetry/context/runtime_context.h
b/api/include/opentelemetry/context/runtime_context.h
+index 5cb793b..24ba44b 100644
+--- a/api/include/opentelemetry/context/runtime_context.h
++++ b/api/include/opentelemetry/context/runtime_context.h
+@@ -4,6 +4,7 @@
+ #pragma once
+
+ #include "opentelemetry/context/context.h"
++#include "opentelemetry/common/threadlocal.h"
+
+ OPENTELEMETRY_BEGIN_NAMESPACE
+ namespace context
+@@ -178,7 +179,7 @@ class ThreadLocalContextStorage : public
RuntimeContextStorage
+ ThreadLocalContextStorage() noexcept = default;
+
+ // Return the current context.
+- Context GetCurrent() noexcept override { return GetStack().Top(); }
++ Context GetCurrent() noexcept override { return GetStack()->Top(); }
+
+ // Resets the context to the value previous to the passed in token. This
will
+ // also detach all child contexts of the passed in token.
+@@ -186,23 +187,23 @@ class ThreadLocalContextStorage : public
RuntimeContextStorage
+ bool Detach(Token &token) noexcept override
+ {
+ // In most cases, the context to be detached is on the top of the stack.
+- if (token == GetStack().Top())
++ if (token == GetStack()->Top())
+ {
+- GetStack().Pop();
++ GetStack()->Pop();
+ return true;
+ }
+
+- if (!GetStack().Contains(token))
++ if (!GetStack()->Contains(token))
+ {
+ return false;
+ }
+
+- while (!(token == GetStack().Top()))
++ while (!(token == GetStack()->Top()))
+ {
+- GetStack().Pop();
++ GetStack()->Pop();
+ }
+
+- GetStack().Pop();
++ GetStack()->Pop();
+
+ return true;
+ }
+@@ -211,14 +212,14 @@ class ThreadLocalContextStorage : public
RuntimeContextStorage
+ // that can be used to reset to the previous Context.
+ nostd::unique_ptr<Token> Attach(const Context &context) noexcept override
+ {
+- GetStack().Push(context);
++ GetStack()->Push(context);
+ return CreateToken(context);
+ }
+
+-private:
+ // A nested class to store the attached contexts in a stack.
+ class Stack
+ {
++ public:
+ friend class ThreadLocalContextStorage;
+
+ Stack() noexcept : size_(0), capacity_(0), base_(nullptr){};
+@@ -305,9 +306,10 @@ class ThreadLocalContextStorage : public
RuntimeContextStorage
+ Context *base_;
+ };
+
+- Stack &GetStack()
++ Stack *GetStack()
+ {
+- static thread_local Stack stack_ = Stack();
++ // static thread_local Stack stack_ = Stack();
++ BLOCK_STATIC_THREAD_LOCAL_TELEMTRY(ThreadLocalContextStorage::Stack,
stack_);
+ return stack_;
+ }
+ };
+diff --git a/sdk/CMakeLists.txt b/sdk/CMakeLists.txt
+index 9aa4588..e2c5a4a 100644
+--- a/sdk/CMakeLists.txt
++++ b/sdk/CMakeLists.txt
+@@ -1,4 +1,4 @@
+-add_library(opentelemetry_sdk INTERFACE)
++add_library(opentelemetry_sdk INTERFACE
../api/include/opentelemetry/common/threadlocal.h)
+ target_include_directories(
+ opentelemetry_sdk
+ INTERFACE "$<BUILD_INTERFACE:${CMAKE_CURRENT_LIST_DIR}/include>"
+diff --git a/sdk/src/common/random.cc b/sdk/src/common/random.cc
+index 77b88cf..dc71f9c 100644
+--- a/sdk/src/common/random.cc
++++ b/sdk/src/common/random.cc
+@@ -3,6 +3,7 @@
+ // SPDX-License-Identifier: Apache-2.0
+
+ #include "src/common/random.h"
++#include "opentelemetry/common/threadlocal.h"
+ #include "src/common/platform/fork.h"
+
+ #include <cstring>
+@@ -29,33 +30,37 @@ class TlsRandomNumberGenerator
+ platform::AtFork(nullptr, nullptr, OnFork);
+ }
+
+- static FastRandomNumberGenerator &engine() noexcept { return engine_; }
++ static FastRandomNumberGenerator *engine() noexcept { return engine_; }
+
+ private:
+- static thread_local FastRandomNumberGenerator engine_;
++ // static thread_local FastRandomNumberGenerator engine_;
++ DECLARE_STATIC_THREAD_LOCAL_TELEMETRY(FastRandomNumberGenerator, engine_);
+
+ static void OnFork() noexcept { Seed(); }
+
+ static void Seed() noexcept
+ {
++ INIT_STATIC_THREAD_LOCAL_TELEMETRY(FastRandomNumberGenerator, engine_);
+ std::random_device random_device;
+ std::seed_seq seed_seq{random_device(), random_device(), random_device(),
random_device()};
+- engine_.seed(seed_seq);
++ engine_->seed(seed_seq);
+ }
+ };
+
+-thread_local FastRandomNumberGenerator TlsRandomNumberGenerator::engine_{};
++// thread_local FastRandomNumberGenerator TlsRandomNumberGenerator::engine_{};
++DEFINE_STATIC_THREAD_LOCAL_TELEMETRY(FastRandomNumberGenerator,
TlsRandomNumberGenerator, engine_);
+ } // namespace
+
+-FastRandomNumberGenerator &Random::GetRandomNumberGenerator() noexcept
++FastRandomNumberGenerator *Random::GetRandomNumberGenerator() noexcept
+ {
+- static thread_local TlsRandomNumberGenerator random_number_generator{};
++ // static thread_local TlsRandomNumberGenerator random_number_generator{};
++ BLOCK_STATIC_THREAD_LOCAL_TELEMTRY(TlsRandomNumberGenerator,
random_number_generator);
+ return TlsRandomNumberGenerator::engine();
+ }
+
+ uint64_t Random::GenerateRandom64() noexcept
+ {
+- return GetRandomNumberGenerator()();
++ return GetRandomNumberGenerator()->operator()();
+ }
+
+ void Random::GenerateRandomBuffer(opentelemetry::nostd::span<uint8_t> buffer)
noexcept
+diff --git a/sdk/src/common/random.h b/sdk/src/common/random.h
+index ecd6dab..1aaa220 100644
+--- a/sdk/src/common/random.h
++++ b/sdk/src/common/random.h
+@@ -34,7 +34,7 @@ class Random
+ /**
+ * @return a seeded thread-local random number generator.
+ */
+- static FastRandomNumberGenerator &GetRandomNumberGenerator() noexcept;
++ static FastRandomNumberGenerator *GetRandomNumberGenerator() noexcept;
+ };
+ } // namespace common
+ } // namespace sdk
diff --git a/thirdparty/vars.sh b/thirdparty/vars.sh
old mode 100755
new mode 100644
index 3adcd3f5b9..1d975bf5cc
--- a/thirdparty/vars.sh
+++ b/thirdparty/vars.sh
@@ -390,6 +390,24 @@ SIMDJSON_NAME=simdjson-1.0.2.tar.gz
SIMDJSON_SOURCE=simdjson-1.0.2
SIMDJSON_MD5SUM="5bb34cca7087a99c450dbdfe406bdc7d"
+# nlohmann_json
+NLOHMANN_JSON_DOWNLOAD="https://github.com/nlohmann/json/archive/refs/tags/v3.10.1.tar.gz";
+NLOHMANN_JSON_NAME=json-3.10.1.tar.gz
+NLOHMANN_JSON_SOURCE=json-3.10.1
+NLOHMANN_JSON_MD5SUM="7b369d567afc0dffdcf5800fd9abb836"
+
+# opentelemetry-proto
+OPENTELEMETRY_PROTO_DOWNLOAD="https://github.com/open-telemetry/opentelemetry-proto/archive/refs/tags/v0.18.0.tar.gz";
+OPENTELEMETRY_PROTO_NAME=opentelemetry-proto-0.18.0.tar.gz
+OPENTELEMETRY_PROTO_SOURCE=opentelemetry-proto-0.18.0
+OPENTELEMETRY_PROTO_MD5SUM="5179f58bb4edbd805590bffd2cf4df85"
+
+# opentelemetry
+OPENTELEMETRY_DOWNLOAD="https://github.com/open-telemetry/opentelemetry-cpp/archive/refs/tags/v1.4.0.tar.gz";
+OPENTELEMETRY_NAME=opentelemetry-cpp-1.4.0.tar.gz
+OPENTELEMETRY_SOURCE=opentelemetry-cpp-1.4.0
+OPENTELEMETRY_MD5SUM="511b670dd1abb596da53684d23742c5f"
+
# libbacktrace
LIBBACKTRACE_DOWNLOAD="https://codeload.github.com/ianlancetaylor/libbacktrace/zip/2446c66076480ce07a6bd868badcbceb3eeecc2e";
LIBBACKTRACE_NAME=libbacktrace-2446c66076480ce07a6bd868badcbceb3eeecc2e.zip
@@ -452,4 +470,7 @@ PDQSORT
BENCHMARK
XSIMD
SIMDJSON
+NLOHMANN_JSON
+OPENTELEMETRY_PROTO
+OPENTELEMETRY
LIBBACKTRACE"
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
