This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 1509fbd538765d549d3cc5cad40e46e64ab90192
Author: FreeOnePlus <54164178+freeonep...@users.noreply.github.com>
AuthorDate: Mon Mar 11 19:09:52 2024 +0800
[feature](docker)Doris On Docker All In One (#31735)
* Refactor upgrade documentation
* Doris On Docker All In One
---------
Co-authored-by: Yijia Su <suyi...@selectdb.com>
---
docker/runtime/all-in-one/Dockerfile | 55 +++++
docker/runtime/all-in-one/resource/entry_point.sh | 248 ++++++++++++++++++++++
2 files changed, 303 insertions(+)
diff --git a/docker/runtime/all-in-one/Dockerfile
b/docker/runtime/all-in-one/Dockerfile
new file mode 100644
index 00000000000..a7bcd98cfed
--- /dev/null
+++ b/docker/runtime/all-in-one/Dockerfile
@@ -0,0 +1,55 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# how to use Dockerfile.
+# this is dockerfile for build doris fe image on amd64.
+# when build youself image.
+# 1. pull binary from official website and decompress into resource directory
that the level equals with Dockerfile_be_ubuntu.
+# 2. untar xxxx.tar.gz in resource directory, update the dockerfile field
`apache-doris-xxx`, replace with real version.
+# 3. run commad docker build -t xxx.doris.be:xx -f Dockerfile_be_ubuntu.
+
+# we have support buildx for amd64 and arm64 architecture image build.
+# get the binary from doris github and utar into resource, update the
directory as apache-`version(example:2.0.1)`-bin-`architecture(amd64/arm64)`
mode.
+
+# choose a base image
+FROM ubuntu:22.04
+
+ARG TARGETARCH
+
+RUN apt-get update -y && DEBIAN_FRONTEND=noninteractive apt-get install -y
--no-install-recommends \
+ patchelf gdb binutils binutils-common mysql-client \
+ curl wget less vim htop iproute2 numactl jq iotop sysstat \
+ tcpdump iputils-ping dnsutils strace lsof blktrace tzdata \
+ bpfcc-tools linux-headers-realtime linux-tools-realtime silversearcher-ag \
+ net-tools openjdk-8-jdk && \
+ rm -rf /var/lib/apt/lists/*
+
+# set environment variables
+ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-${TARGETARCH:-amd64} \
+
PATH="/opt/apache-doris/fe/bin:/opt/apache-doris/be/bin:/opt/apache-doris/broker/bin:${PATH}"
+
+# apache-doris/be from doris release xxxx.tar.gz.please update the version in
follows x.x.x.
+ADD resource/apache-doris-x.x.x-bin-${TARGETARCH:-amd64}/fe
/opt/apache-doris/fe
+ADD resource/apache-doris-x.x.x-bin-${TARGETARCH:-amd64}/be
/opt/apache-doris/be
+ADD resource/apache-doris-x.x.x-bin-${TARGETARCH:-amd64}/broker
/opt/apache-doris/broker
+
+COPY resource/entry_point.sh /usr/local/bin/
+
+WORKDIR /opt/apache-doris
+
+ENTRYPOINT ["bash","entry_point.sh"]
diff --git a/docker/runtime/all-in-one/resource/entry_point.sh
b/docker/runtime/all-in-one/resource/entry_point.sh
new file mode 100644
index 00000000000..b0ec4882977
--- /dev/null
+++ b/docker/runtime/all-in-one/resource/entry_point.sh
@@ -0,0 +1,248 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -eo pipefail
+shopt -s nullglob
+
+DORIS_HOME="/opt/apache-doris"
+
+# Obtain necessary and basic information to complete initialization
+
+# logging functions
+# usage: doris_[note|warn|error] $log_meg
+# ie: doris_warn "task may be risky!"
+# out: 2023-01-08T19:08:16+08:00 [Warn] [Entrypoint]: task may be risky!
+doris_log() {
+ local type="$1"
+ shift
+ # accept argument string or stdin
+ local text="$*"
+ if [ "$#" -eq 0 ]; then text="$(cat)"; fi
+ local dt="$(date -Iseconds)"
+ printf '%s [%s] [Entrypoint]: %s\n' "$dt" "$type" "$text"
+}
+doris_note() {
+ doris_log Note "$@"
+}
+doris_warn() {
+ doris_log Warn "$@" >&2
+}
+doris_error() {
+ doris_log ERROR "$@" >&2
+ exit 1
+}
+
+# check to see if this file is being run or sourced from another script
+_is_sourced() {
+ [ "${#FUNCNAME[@]}" -ge 2 ] &&
+ [ "${FUNCNAME[0]}" = '_is_sourced' ] &&
+ [ "${FUNCNAME[1]}" = 'source' ]
+}
+
+docker_setup_env() {
+ declare -g METADATA_FAILURE_RECOVERY MASTER_FE_IP CURRENT_BE_IP \
+ CURRENT_BE_PORT CURRENT_BROKER_IP CURRENT_BROKER_PORT
DATABASE_ALREADY_EXISTS
+ MASTER_FE_IP="127.0.0.1"
+ CURRENT_BE_IP="127.0.0.1"
+ CURRENT_BE_PORT=9050
+ CURRENT_BROKER_IP="127.0.0.1"
+ CURRENT_BROKER_PORT=8000
+ if [[ $RECOVERY == "true" ]]; then
+ METADATA_FAILURE_RECOVERY='true'
+ fi
+ if [ -d "${DORIS_HOME}/fe/doris-meta/image" ]; then
+ DATABASE_ALREADY_EXISTS='true'
+ fi
+}
+
+# Execute sql script, passed via stdin
+docker_process_sql() {
+ set +e
+ mysql -uroot -P9030 -h127.0.0.1 --comments "$@" 2>&1
+}
+
+check_be_status() {
+ set +e
+ declare -g BE_ALREADY_EXISTS
+ for i in {1..300}; do
+ if [[ $1 == true ]]; then
+ docker_process_sql <<<"show frontends" | grep
"[[:space:]]${MASTER_FE_IP}[[:space:]]"
+ else
+ docker_process_sql <<<"show backends" | grep
"[[:space:]]${CURRENT_BE_IP}[[:space:]]" | grep
"[[:space:]]${CURRENT_BE_PORT}[[:space:]]" | grep "[[:space:]]true[[:space:]]"
+ fi
+ be_join_status=$?
+ if [[ "${be_join_status}" == 0 ]]; then
+ if [[ $1 == true ]]; then
+ doris_note "MASTER FE is started!"
+ else
+ doris_note "EntryPoint Check - Verify that BE is registered to FE
successfully"
+ BE_ALREADY_EXISTS=true
+ fi
+ return
+ fi
+ if [[ $(( $i % 20 )) == 1 ]]; then
+ if [[ $1 == true ]]; then
+ doris_note "MASTER FE is not started. retry."
+ else
+ doris_note "BE is not register. retry."
+ fi
+ fi
+ sleep 1
+ done
+}
+
+check_broker_status() {
+ set +e
+ declare -g BROKER_ALREADY_EXISTS
+ if [[ $1 == true ]]; then
+ docker_process_sql <<<"show broker" | grep
"[[:space:]]${MASTER_FE_IP}[[:space:]]"
+ else
+ docker_process_sql <<<"show broker" | grep
"[[:space:]]${CURRENT_BROKER_IP}[[:space:]]" | grep
"[[:space:]]${CURRENT_BROKER_PORT}[[:space:]]" | grep
"[[:space:]]true[[:space:]]"
+ fi
+ broker_join_status=$?
+ if [[ "${broker_join_status}" == 0 ]]; then
+ if [[ $1 == true ]]; then
+ doris_note "MASTER FE is started!"
+ else
+ doris_note "EntryPoint Check - Verify that Broker is registered to FE
successfully"
+ BROKER_ALREADY_EXISTS=true
+ fi
+ return
+ fi
+ sleep 1
+}
+
+add_priority_networks() {
+ doris_note "add priority_networks ‘127.0.0.1/24’ to
${DORIS_HOME}/be/conf/be.conf"
+ echo "priority_networks = 127.0.0.1/24" >>${DORIS_HOME}/be/conf/be.conf
+}
+
+register_be_to_fe() {
+ set +e
+ # check fe status
+ local is_fe_start=false
+ if [ -n "$DATABASE_ALREADY_EXISTS" ]; then
+ check_be_status
+ if [ -n "$BE_ALREADY_EXISTS" ]; then
+ doris_warn "Same backend already exists! No need to register again！"
+ return
+ fi
+ fi
+ for i in {1..300}; do
+ docker_process_sql <<<"alter system add backend
'${CURRENT_BE_IP}:${CURRENT_BE_PORT}'"
+ register_be_status=$?
+ if [[ $register_be_status == 0 ]]; then
+ doris_note "BE successfully registered to FE！"
+ is_fe_start=true
+ return
+ fi
+ if [[ $(( $i % 20 )) == 1 ]]; then
+ doris_note "Register BE to FE is failed. retry."
+ fi
+ sleep 1
+ done
+ if ! [[ $is_fe_start ]]; then
+ doris_error "Failed to register BE to FE！Tried 30 times！Maybe FE Start
Failed！"
+ fi
+}
+
+register_broker_to_fe() {
+ set +e
+ # check fe status
+ local is_fe_start=false
+ for i in {1..300}; do
+ if [ -n "$BROKER_ALREADY_EXISTS" ]; then
+ doris_warn "Same Broker already exists! No need to register again！"
+ break
+ fi
+ docker_process_sql <<<"alter system add broker test
'${CURRENT_BROKER_IP}:${CURRENT_BROKER_PORT}'"
+ register_broker_status=$?
+ if [[ $register_broker_status == 0 ]]; then
+ doris_note "Broker successfully registered to FE！"
+ is_fe_start=true
+ break
+ fi
+ if [[ $(( $i % 20 )) == 1 ]]; then
+ doris_note "Register Broker to FE is failed. retry."
+ fi
+ sleep 1
+ done
+ if ! [[ $is_fe_start ]]; then
+ doris_error "Failed to register Broker to FE！Tried 30 times！Maybe FE Start
Failed！"
+ fi
+}
+
+start_doris() {
+ declare -g child_pid
+ if [[ $METADATA_FAILURE_RECOVERY == "true" ]]; then
+ doris_warn "Because \$RECOVERY = True, So Doris FE start
metadata_failure_recovery model."
+ start_fe.sh --metadata_failure_recovery --console
+ else
+ doris_note "Start Doris FE."
+ {
+ set +e
+ bash start_fe.sh --console 2>/dev/null
+ } &
+ fi
+ if [[ $BROKER = "true" ]]; then
+ sleep 20
+ {
+ set +e
+ bash start_broker.sh 2>/dev/null
+ } &
+ check_broker_status
+ register_broker_to_fe
+ fi
+ sleep 20
+ doris_note "Start Doris BE."
+ {
+ set +e
+ bash start_be.sh --console 2>/dev/null
+ } &
+ child_pid=$!
+}
+
+stop_doris() {
+ doris_note "Container stopped, running stop_fe & stop_be script"
+ stop_broker.sh
+ sleep 2
+ stop_be.sh
+ sleep 10
+ stop_fe.sh
+}
+
+_main() {
+ trap 'stop_doris' SIGTERM SIGINT
+ docker_setup_env
+ # Check Already Exists
+ if [ -z "$DATABASE_ALREADY_EXISTS" ]; then
+ add_priority_networks
+ fi
+ # Start Doris
+ start_doris
+ # register BE
+ register_be_to_fe
+ # keep BE started status
+ wait $child_pid
+ doris_note "Apache Doris is Start Successfully!"
+ exec "$@"
+}
+
+if ! _is_sourced; then
+ _main "$@"
+fi
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
