This is an automated email from the ASF dual-hosted git repository. yangzhg pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push: new 12652ebb0e [UDF](java udf) using config to enable java udf instead of macro at compile time (#14062) 12652ebb0e is described below commit 12652ebb0ea6be6be8e838efa4f5765e63dc97a6 Author: Zhengguo Yang <yangz...@gmail.com> AuthorDate: Fri Nov 11 09:03:52 2022 +0800 [UDF](java udf) using config to enable java udf instead of macro at compile time (#14062) * [UDF](java udf) useing config to enable java udf instead of macro at compile time --- be/CMakeLists.txt | 6 +-- be/src/common/config.h | 3 ++ be/src/exec/data_sink.cpp | 12 ++--- be/src/exec/exec_node.cpp | 19 ++++---- be/src/runtime/plan_fragment_executor.cpp | 7 +-- be/src/runtime/user_function_cache.cpp | 51 +++++++++++----------- be/src/service/doris_main.cpp | 14 +++--- be/src/util/CMakeLists.txt | 5 +-- be/src/util/jni-util.cpp | 2 - be/src/util/jni-util.h | 3 -- .../aggregate_function_java_udaf.h | 3 -- be/src/vec/exec/scan/new_jdbc_scan_node.cpp | 2 - be/src/vec/exec/scan/new_jdbc_scan_node.h | 2 - be/src/vec/exec/scan/new_jdbc_scanner.cpp | 3 -- be/src/vec/exec/scan/new_jdbc_scanner.h | 2 - be/src/vec/exec/vjdbc_connector.cpp | 4 +- be/src/vec/exec/vjdbc_connector.h | 6 +-- be/src/vec/exprs/vectorized_agg_fn.cpp | 12 ++--- be/src/vec/exprs/vectorized_fn_call.cpp | 12 ++--- be/src/vec/functions/function_java_udf.cpp | 2 - be/src/vec/functions/function_java_udf.h | 2 - be/src/vec/sink/vjdbc_table_sink.cpp | 2 - be/src/vec/sink/vjdbc_table_sink.h | 2 - build.sh | 1 - 24 files changed, 73 insertions(+), 104 deletions(-) diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt index 9a79ca4994..e0d0ba8b7d 100644 --- a/be/CMakeLists.txt +++ b/be/CMakeLists.txt @@ -600,17 +600,13 @@ include_directories( ${GENSRC_DIR}/ ${THIRDPARTY_DIR}/include ${GPERFTOOLS_HOME}/include -) - -if (BUILD_JAVA_UDF) include_directories($ENV{JAVA_HOME}/include) if (NOT OS_MACOSX) include_directories($ENV{JAVA_HOME}/include/linux) else() include_directories($ENV{JAVA_HOME}/include/darwin) endif() - add_definitions("-DLIBJVM") -endif() +) if (NOT OS_MACOSX) set(WL_START_GROUP "-Wl,--start-group") diff --git a/be/src/common/config.h b/be/src/common/config.h index bc3c67d92c..dbc58650f5 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -850,6 +850,9 @@ CONF_Int32(segcompaction_small_threshold, "1048576"); CONF_String(jvm_max_heap_size, "1024M"); +// enable java udf and jdbc scannode +CONF_Bool(enable_java_support, "true"); + #ifdef BE_TEST // test s3 CONF_String(test_s3_resource, "resource"); diff --git a/be/src/exec/data_sink.cpp b/be/src/exec/data_sink.cpp index cb309bd0d8..0c8842ad32 100644 --- a/be/src/exec/data_sink.cpp +++ b/be/src/exec/data_sink.cpp @@ -171,11 +171,13 @@ Status DataSink::create_data_sink(ObjectPool* pool, const TDataSink& thrift_sink return Status::InternalError("Missing data jdbc sink."); } if (is_vec) { -#ifdef LIBJVM - sink->reset(new vectorized::VJdbcTableSink(pool, row_desc, output_exprs)); -#else - return Status::InternalError("Jdbc table sink is disabled since no libjvm is found!"); -#endif + if (config::enable_java_support) { + sink->reset(new vectorized::VJdbcTableSink(pool, row_desc, output_exprs)); + } else { + return Status::InternalError( + "Jdbc table sink is not enabled, you can change be config " + "enable_java_support to true and restart be."); + } } else { return Status::InternalError("only support jdbc sink in vectorized engine."); } diff --git a/be/src/exec/exec_node.cpp b/be/src/exec/exec_node.cpp index 98ce9d9950..97ccf5f701 100644 --- a/be/src/exec/exec_node.cpp +++ b/be/src/exec/exec_node.cpp @@ -450,11 +450,13 @@ Status ExecNode::create_node(RuntimeState* state, ObjectPool* pool, const TPlanN case TPlanNodeType::JDBC_SCAN_NODE: if (state->enable_vectorized_exec()) { -#ifdef LIBJVM - *node = pool->add(new vectorized::NewJdbcScanNode(pool, tnode, descs)); -#else - return Status::InternalError("Jdbc scan node is disabled since no libjvm is found!"); -#endif + if (config::enable_java_support) { + *node = pool->add(new vectorized::NewJdbcScanNode(pool, tnode, descs)); + } else { + return Status::InternalError( + "Jdbc scan node is disabled, you can change be config enable_java_support " + "to true and restart be."); + } } else { return Status::InternalError("Jdbc scan node only support vectorized engine."); } @@ -722,11 +724,8 @@ void ExecNode::try_do_aggregate_serde_improve() { if (typeid(*child0) == typeid(vectorized::NewOlapScanNode) || typeid(*child0) == typeid(vectorized::NewFileScanNode) || typeid(*child0) == typeid(vectorized::NewOdbcScanNode) || - typeid(*child0) == typeid(vectorized::NewEsScanNode) -#ifdef LIBJVM - || typeid(*child0) == typeid(vectorized::NewJdbcScanNode) -#endif - ) { + typeid(*child0) == typeid(vectorized::NewEsScanNode) || + typeid(*child0) == typeid(vectorized::NewJdbcScanNode)) { vectorized::VScanNode* scan_node = static_cast<vectorized::VScanNode*>(agg_node[0]->_children[0]); scan_node->set_no_agg_finalize(); diff --git a/be/src/runtime/plan_fragment_executor.cpp b/be/src/runtime/plan_fragment_executor.cpp index 5f83fe454a..8209c82e83 100644 --- a/be/src/runtime/plan_fragment_executor.cpp +++ b/be/src/runtime/plan_fragment_executor.cpp @@ -170,11 +170,8 @@ Status PlanFragmentExecutor::prepare(const TExecPlanFragmentParams& request, if (typeid(*node) == typeid(vectorized::NewOlapScanNode) || typeid(*node) == typeid(vectorized::NewFileScanNode) || typeid(*node) == typeid(vectorized::NewOdbcScanNode) || - typeid(*node) == typeid(vectorized::NewEsScanNode) -#ifdef LIBJVM - || typeid(*node) == typeid(vectorized::NewJdbcScanNode) -#endif - ) { + typeid(*node) == typeid(vectorized::NewEsScanNode) || + typeid(*node) == typeid(vectorized::NewJdbcScanNode)) { vectorized::VScanNode* scan_node = static_cast<vectorized::VScanNode*>(scan_nodes[i]); const std::vector<TScanRangeParams>& scan_ranges = find_with_default(params.per_node_scan_ranges, scan_node->id(), no_scan_ranges); diff --git a/be/src/runtime/user_function_cache.cpp b/be/src/runtime/user_function_cache.cpp index b985fec75d..475a8a52e8 100644 --- a/be/src/runtime/user_function_cache.cpp +++ b/be/src/runtime/user_function_cache.cpp @@ -21,17 +21,16 @@ #include <regex> #include <vector> +#include "common/config.h" #include "env/env.h" #include "gutil/strings/split.h" #include "http/http_client.h" #include "util/dynamic_util.h" #include "util/file_utils.h" -#include "util/string_util.h" -#ifdef LIBJVM #include "util/jni-util.h" -#endif #include "util/md5.h" #include "util/spinlock.h" +#include "util/string_util.h" namespace doris { @@ -377,28 +376,30 @@ Status UserFunctionCache::_load_cache_entry_internal(UserFunctionCacheEntry* ent } Status UserFunctionCache::_add_to_classpath(UserFunctionCacheEntry* entry) { -#ifdef LIBJVM - const std::string path = "file://" + entry->lib_file; - LOG(INFO) << "Add jar " << path << " to classpath"; - JNIEnv* env; - RETURN_IF_ERROR(JniUtil::GetJNIEnv(&env)); - jclass class_class_loader = env->FindClass("java/lang/ClassLoader"); - jmethodID method_get_system_class_loader = env->GetStaticMethodID( - class_class_loader, "getSystemClassLoader", "()Ljava/lang/ClassLoader;"); - jobject class_loader = - env->CallStaticObjectMethod(class_class_loader, method_get_system_class_loader); - jclass class_url_class_loader = env->FindClass("java/net/URLClassLoader"); - jmethodID method_add_url = - env->GetMethodID(class_url_class_loader, "addURL", "(Ljava/net/URL;)V"); - jclass class_url = env->FindClass("java/net/URL"); - jmethodID url_ctor = env->GetMethodID(class_url, "<init>", "(Ljava/lang/String;)V"); - jobject urlInstance = env->NewObject(class_url, url_ctor, env->NewStringUTF(path.c_str())); - env->CallVoidMethod(class_loader, method_add_url, urlInstance); - entry->is_loaded.store(true); - return Status::OK(); -#else - return Status::InternalError("No libjvm is found!"); -#endif + if (config::enable_java_support) { + const std::string path = "file://" + entry->lib_file; + LOG(INFO) << "Add jar " << path << " to classpath"; + JNIEnv* env; + RETURN_IF_ERROR(JniUtil::GetJNIEnv(&env)); + jclass class_class_loader = env->FindClass("java/lang/ClassLoader"); + jmethodID method_get_system_class_loader = env->GetStaticMethodID( + class_class_loader, "getSystemClassLoader", "()Ljava/lang/ClassLoader;"); + jobject class_loader = + env->CallStaticObjectMethod(class_class_loader, method_get_system_class_loader); + jclass class_url_class_loader = env->FindClass("java/net/URLClassLoader"); + jmethodID method_add_url = + env->GetMethodID(class_url_class_loader, "addURL", "(Ljava/net/URL;)V"); + jclass class_url = env->FindClass("java/net/URL"); + jmethodID url_ctor = env->GetMethodID(class_url, "<init>", "(Ljava/lang/String;)V"); + jobject urlInstance = env->NewObject(class_url, url_ctor, env->NewStringUTF(path.c_str())); + env->CallVoidMethod(class_loader, method_add_url, urlInstance); + entry->is_loaded.store(true); + return Status::OK(); + } else { + return Status::InternalError( + "Java UDF is not enabled, you can change be config enable_java_support to true and " + "restart be."); + } } std::string UserFunctionCache::_make_lib_file(int64_t function_id, const std::string& checksum, diff --git a/be/src/service/doris_main.cpp b/be/src/service/doris_main.cpp index 714e4757bc..5d660a685e 100644 --- a/be/src/service/doris_main.cpp +++ b/be/src/service/doris_main.cpp @@ -375,14 +375,14 @@ int main(int argc, char** argv) { apache::thrift::GlobalOutput.setOutputFunction(doris::thrift_output); Status status = Status::OK(); -#ifdef LIBJVM - // Init jni - status = doris::JniUtil::Init(); - if (!status.ok()) { - LOG(WARNING) << "Failed to initialize JNI: " << status.get_error_msg(); - exit(1); + if (doris::config::enable_java_support) { + // Init jni + status = doris::JniUtil::Init(); + if (!status.ok()) { + LOG(WARNING) << "Failed to initialize JNI: " << status.get_error_msg(); + exit(1); + } } -#endif doris::Daemon daemon; daemon.init(argc, argv, paths); diff --git a/be/src/util/CMakeLists.txt b/be/src/util/CMakeLists.txt index dc9509a9ec..6fe40c4bcd 100644 --- a/be/src/util/CMakeLists.txt +++ b/be/src/util/CMakeLists.txt @@ -114,6 +114,7 @@ set(UTIL_FILES quantile_state.cpp jni-util.cpp exception.cpp + libjvm_loader.cpp ) if (WITH_MYSQL) @@ -127,10 +128,6 @@ if (OS_MACOSX) list(APPEND UTIL_FILES perf_counters_mac.cpp disk_info_mac.cpp) endif() -if (BUILD_JAVA_UDF) - list(APPEND UTIL_FILES libjvm_loader.cpp) -endif() - add_library(Util STATIC ${UTIL_FILES} ) diff --git a/be/src/util/jni-util.cpp b/be/src/util/jni-util.cpp index 7c56381052..731bf310e8 100644 --- a/be/src/util/jni-util.cpp +++ b/be/src/util/jni-util.cpp @@ -17,7 +17,6 @@ #include "util/jni-util.h" -#ifdef LIBJVM #include <jni.h> #include <jni_md.h> #include <stdlib.h> @@ -290,4 +289,3 @@ Status JniUtil::Init() { } } // namespace doris -#endif diff --git a/be/src/util/jni-util.h b/be/src/util/jni-util.h index ff3c40609b..5593d77dda 100644 --- a/be/src/util/jni-util.h +++ b/be/src/util/jni-util.h @@ -17,7 +17,6 @@ #pragma once -#ifdef LIBJVM #include <jni.h> #include "common/status.h" @@ -155,5 +154,3 @@ Status SerializeThriftMsg(JNIEnv* env, T* msg, jbyteArray* serialized_msg) { } } // namespace doris - -#endif diff --git a/be/src/vec/aggregate_functions/aggregate_function_java_udaf.h b/be/src/vec/aggregate_functions/aggregate_function_java_udaf.h index 9252357255..ba442f3b79 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_java_udaf.h +++ b/be/src/vec/aggregate_functions/aggregate_function_java_udaf.h @@ -17,8 +17,6 @@ #pragma once -#ifdef LIBJVM - #include <jni.h> #include <unistd.h> @@ -401,4 +399,3 @@ private: }; } // namespace doris::vectorized -#endif diff --git a/be/src/vec/exec/scan/new_jdbc_scan_node.cpp b/be/src/vec/exec/scan/new_jdbc_scan_node.cpp index 5f209a09c2..e86060ad41 100644 --- a/be/src/vec/exec/scan/new_jdbc_scan_node.cpp +++ b/be/src/vec/exec/scan/new_jdbc_scan_node.cpp @@ -16,7 +16,6 @@ // under the License. #include "vec/exec/scan/new_jdbc_scan_node.h" -#ifdef LIBJVM #include "vec/exec/scan/new_jdbc_scanner.h" #include "vec/exec/scan/vscanner.h" @@ -58,4 +57,3 @@ Status NewJdbcScanNode::_init_scanners(std::list<VScanner*>* scanners) { return Status::OK(); } } // namespace doris::vectorized -#endif diff --git a/be/src/vec/exec/scan/new_jdbc_scan_node.h b/be/src/vec/exec/scan/new_jdbc_scan_node.h index 7463e55daf..d527f24b12 100644 --- a/be/src/vec/exec/scan/new_jdbc_scan_node.h +++ b/be/src/vec/exec/scan/new_jdbc_scan_node.h @@ -16,7 +16,6 @@ // under the License. #pragma once -#ifdef LIBJVM #include "runtime/runtime_state.h" #include "vec/exec/scan/vscan_node.h" @@ -41,4 +40,3 @@ private: }; } // namespace vectorized } // namespace doris -#endif diff --git a/be/src/vec/exec/scan/new_jdbc_scanner.cpp b/be/src/vec/exec/scan/new_jdbc_scanner.cpp index 2141d6b7e8..066074f16a 100644 --- a/be/src/vec/exec/scan/new_jdbc_scanner.cpp +++ b/be/src/vec/exec/scan/new_jdbc_scanner.cpp @@ -17,8 +17,6 @@ #include "new_jdbc_scanner.h" -#ifdef LIBJVM - namespace doris::vectorized { NewJdbcScanner::NewJdbcScanner(RuntimeState* state, NewJdbcScanNode* parent, int64_t limit, TupleId tuple_id, std::string query_string) @@ -151,4 +149,3 @@ Status NewJdbcScanner::close(RuntimeState* state) { return Status::OK(); } } // namespace doris::vectorized -#endif diff --git a/be/src/vec/exec/scan/new_jdbc_scanner.h b/be/src/vec/exec/scan/new_jdbc_scanner.h index 984e239c2d..6a45462db1 100644 --- a/be/src/vec/exec/scan/new_jdbc_scanner.h +++ b/be/src/vec/exec/scan/new_jdbc_scanner.h @@ -16,7 +16,6 @@ // under the License. #pragma once -#ifdef LIBJVM #include "runtime/runtime_state.h" #include "vec/exec/scan/new_jdbc_scan_node.h" @@ -55,4 +54,3 @@ private: }; } // namespace vectorized } // namespace doris -#endif diff --git a/be/src/vec/exec/vjdbc_connector.cpp b/be/src/vec/exec/vjdbc_connector.cpp index 283aa3e2a7..b2f730ed14 100644 --- a/be/src/vec/exec/vjdbc_connector.cpp +++ b/be/src/vec/exec/vjdbc_connector.cpp @@ -16,7 +16,7 @@ // under the License. #include "vec/exec/vjdbc_connector.h" -#ifdef LIBJVM + #include "common/status.h" #include "exec/table_connector.h" #include "gen_cpp/Types_types.h" @@ -409,5 +409,3 @@ FUNC_IMPL_TO_CONVERT_DATA(double, double, D, Double) } // namespace vectorized } // namespace doris - -#endif diff --git a/be/src/vec/exec/vjdbc_connector.h b/be/src/vec/exec/vjdbc_connector.h index ba2f19c7b8..f3575dfd52 100644 --- a/be/src/vec/exec/vjdbc_connector.h +++ b/be/src/vec/exec/vjdbc_connector.h @@ -16,10 +16,10 @@ // under the License. #pragma once -#ifdef LIBJVM + +#include <jni.h> #include "exec/table_connector.h" -#include "jni.h" namespace doris { namespace vectorized { @@ -104,5 +104,3 @@ private: } // namespace vectorized } // namespace doris - -#endif diff --git a/be/src/vec/exprs/vectorized_agg_fn.cpp b/be/src/vec/exprs/vectorized_agg_fn.cpp index dc88b7f07f..d47d9d32be 100644 --- a/be/src/vec/exprs/vectorized_agg_fn.cpp +++ b/be/src/vec/exprs/vectorized_agg_fn.cpp @@ -114,11 +114,13 @@ Status AggFnEvaluator::prepare(RuntimeState* state, const RowDescriptor& desc, M _real_argument_types.empty() ? tmp_argument_types : _real_argument_types; if (_fn.binary_type == TFunctionBinaryType::JAVA_UDF) { -#ifdef LIBJVM - _function = AggregateJavaUdaf::create(_fn, argument_types, {}, _data_type); -#else - return Status::InternalError("Java UDAF is disabled since no libjvm is found!"); -#endif + if (config::enable_java_support) { + _function = AggregateJavaUdaf::create(_fn, argument_types, {}, _data_type); + } else { + return Status::InternalError( + "Java UDF is not enabled, you can change be config enable_java_support to true " + "and restart be."); + } } else if (_fn.binary_type == TFunctionBinaryType::RPC) { _function = AggregateRpcUdaf::create(_fn, argument_types, {}, _data_type); } else { diff --git a/be/src/vec/exprs/vectorized_fn_call.cpp b/be/src/vec/exprs/vectorized_fn_call.cpp index 1bf05ce00e..c535ca266f 100644 --- a/be/src/vec/exprs/vectorized_fn_call.cpp +++ b/be/src/vec/exprs/vectorized_fn_call.cpp @@ -58,11 +58,13 @@ doris::Status VectorizedFnCall::prepare(doris::RuntimeState* state, if (_fn.binary_type == TFunctionBinaryType::RPC) { _function = FunctionRPC::create(_fn, argument_template, _data_type); } else if (_fn.binary_type == TFunctionBinaryType::JAVA_UDF) { -#ifdef LIBJVM - _function = JavaFunctionCall::create(_fn, argument_template, _data_type); -#else - return Status::InternalError("Java UDF is disabled since no libjvm is found!"); -#endif + if (config::enable_java_support) { + _function = JavaFunctionCall::create(_fn, argument_template, _data_type); + } else { + return Status::InternalError( + "Java UDF is not enabled, you can change be config enable_java_support to true " + "and restart be."); + } } else { _function = SimpleFunctionFactory::instance().get_function(_fn.name.function_name, argument_template, _data_type); diff --git a/be/src/vec/functions/function_java_udf.cpp b/be/src/vec/functions/function_java_udf.cpp index 1a2a52a515..d5e3751ef5 100644 --- a/be/src/vec/functions/function_java_udf.cpp +++ b/be/src/vec/functions/function_java_udf.cpp @@ -17,7 +17,6 @@ #include "vec/functions/function_java_udf.h" -#ifdef LIBJVM #include <fmt/format.h> #include <memory> @@ -223,4 +222,3 @@ Status JavaFunctionCall::close(FunctionContext* context, return Status::OK(); } } // namespace doris::vectorized -#endif diff --git a/be/src/vec/functions/function_java_udf.h b/be/src/vec/functions/function_java_udf.h index 8d878ff2b8..abb06e4d38 100644 --- a/be/src/vec/functions/function_java_udf.h +++ b/be/src/vec/functions/function_java_udf.h @@ -17,7 +17,6 @@ #pragma once -#ifdef LIBJVM #include <jni.h> #include "gen_cpp/Exprs_types.h" @@ -128,4 +127,3 @@ private: } // namespace vectorized } // namespace doris -#endif diff --git a/be/src/vec/sink/vjdbc_table_sink.cpp b/be/src/vec/sink/vjdbc_table_sink.cpp index 4c428aa47b..0d9e457cff 100644 --- a/be/src/vec/sink/vjdbc_table_sink.cpp +++ b/be/src/vec/sink/vjdbc_table_sink.cpp @@ -17,7 +17,6 @@ #include "vec/sink/vjdbc_table_sink.h" -#ifdef LIBJVM #include <gen_cpp/DataSinks_types.h> #include <sstream> @@ -101,4 +100,3 @@ Status VJdbcTableSink::close(RuntimeState* state, Status exec_status) { } } // namespace vectorized } // namespace doris -#endif \ No newline at end of file diff --git a/be/src/vec/sink/vjdbc_table_sink.h b/be/src/vec/sink/vjdbc_table_sink.h index 9f4d42ad5a..52238942a2 100644 --- a/be/src/vec/sink/vjdbc_table_sink.h +++ b/be/src/vec/sink/vjdbc_table_sink.h @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. #pragma once -#ifdef LIBJVM #include "common/status.h" #include "vec/exec/vjdbc_connector.h" #include "vec/sink/vtable_sink.h" @@ -45,4 +44,3 @@ private: }; } // namespace vectorized } // namespace doris -#endif \ No newline at end of file diff --git a/build.sh b/build.sh index aba8a7f565..8213464adb 100755 --- a/build.sh +++ b/build.sh @@ -406,7 +406,6 @@ if [[ "${BUILD_BE}" -eq 1 ]]; then -DWITH_LZO="${WITH_LZO}" \ -DUSE_LIBCPP="${USE_LIBCPP}" \ -DBUILD_META_TOOL="${BUILD_META_TOOL}" \ - -DBUILD_JAVA_UDF="${BUILD_JAVA_UDF}" \ -DSTRIP_DEBUG_INFO="${STRIP_DEBUG_INFO}" \ -DUSE_DWARF="${USE_DWARF}" \ -DUSE_MEM_TRACKER="${USE_MEM_TRACKER}" \ --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org