This is an automated email from the ASF dual-hosted git repository.
mcvsubbu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new f3ed95c Add compatibility verifier scripts (#4854) (#5911)
f3ed95c is described below
commit f3ed95cfd5d8733a2dda8354e80f03fd7691523f
Author: Sailesh Nankani <30644225+saileshnank...@users.noreply.github.com>
AuthorDate: Wed Sep 2 16:31:45 2020 -0400
Add compatibility verifier scripts (#4854) (#5911)
* Add compatibility verifier scripts (#4854)
Adds 3 scripts to take 2 commit hashes and verify compatibility across
these.
- build-release.sh
- comp-verifier.sh
- control-service.sh
Starts all components in v1 version. Upgrades controller first, then the
broker, and finally the server.
* Format and address review comments
* Add error handling and comments
* Address further PR comments
* Fix additional PR comments
---
compatibility-verifier/compCheck.sh | 177 ++++++++++++++++++++++++++++++++++++
1 file changed, 177 insertions(+)
diff --git a/compatibility-verifier/compCheck.sh
b/compatibility-verifier/compCheck.sh
new file mode 100755
index 0000000..af81744
--- /dev/null
+++ b/compatibility-verifier/compCheck.sh
@@ -0,0 +1,177 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# A script that does rolling upgrade of Pinot components
+# from one version to the other given 2 commit hashes. It first builds
+# Pinot in the 2 given directories and then upgrades in the following order:
+# Controller -> Broker -> Server
+
+# get a temporary directory in case the workingDir is not provided by user
+TMP_DIR=$(mktemp -d 2>/dev/null || mktemp -d -t 'mytmpdir')
+
+# get usage of the script
+function usage() {
+ command=$1
+ echo "Usage: $command olderCommit newerCommit [workingDir]"
+ exit 1
+}
+
+# cleanup the temporary directory when exiting the script
+function cleanup() {
+ if [ -n "$TMP_DIR" ] && [ -d "$TMP_DIR" ] && [ "$workingDir" = "$TMP_DIR" ]
; then
+ echo "The temporary directory $TMP_DIR needs to be cleaned up."
+ fi
+}
+
+# This function builds Pinot given a specific commit hash and target directory
+function checkoutAndBuild() {
+ commitHash=$1
+ targetDir=$2
+
+ pushd "$targetDir" || exit 1
+ git init
+ git remote add origin https://github.com/apache/incubator-pinot
+ git fetch --depth 1 origin "$commitHash"
+ git checkout FETCH_HEAD
+ mvn install package -DskipTests -Pbin-dist
+ popd || exit 1
+}
+
+# Given a component and directory, start that version of the specific
component
+function startService() {
+ serviceName=$1
+ dirName=$2
+ # Upon start, save the pid of the process for a component into a file in
/tmp/{component}.pid, which is then used to stop it
+ pushd "$dirName"/pinot-tools/target/pinot-tools-pkg/bin || exit 1
+ if [ "$serviceName" = "zookeeper" ]; then
+ sh -c 'echo $$ > $0/zookeeper.pid; exec ./pinot-admin.sh StartZookeeper'
"${dirName}" &
+ elif [ "$serviceName" = "controller" ]; then
+ sh -c 'echo $$ > $0/controller.pid; exec ./pinot-admin.sh StartController'
"${dirName}" &
+ elif [ "$serviceName" = "broker" ]; then
+ sh -c 'echo $$ > $0/broker.pid; exec ./pinot-admin.sh StartBroker'
"${dirName}" &
+ elif [ "$serviceName" = "server" ]; then
+ sh -c 'echo $$ > $0/server.pid; exec ./pinot-admin.sh StartServer'
"${dirName}" &
+ fi
+ popd || exit 1
+}
+
+# Given a component, check if it known to be running and stop that specific
component
+function stopService() {
+ serviceName=$1
+ dirName=$2
+ if [ -f "${dirName}/${serviceName}".pid ]; then
+ servicePid=$(<"${dirName}/${serviceName}".pid)
+ rm "${dirName}/${serviceName}".pid
+ if [ -n "$servicePid" ]; then
+ kill -9 "$servicePid"
+ fi
+ else
+ echo "Pid file ${dirName}/${serviceName}.pid not found. Failed to stop
component ${serviceName}"
+ fi
+}
+
+# Starts a Pinot cluster given a specific target directory
+function startServices() {
+ dirName=$1
+ startService zookeeper "$dirName"
+ startService controller "$dirName"
+ startService broker "$dirName"
+ startService server "$dirName"
+ echo "Cluster started."
+}
+
+# Stops the currently running Pinot cluster
+function stopServices() {
+ dirName=$1
+ stopService controller "$dirName"
+ stopService broker "$dirName"
+ stopService server "$dirName"
+ stopService zookeeper "$dirName"
+ echo "Cluster stopped."
+}
+
+# cleanp the temporary directory when the bash script exits
+trap cleanup EXIT
+
+if [ $# -lt 2 ] || [ $# -gt 3 ] ; then
+ usage compCheck
+fi
+
+# get arguments
+olderCommit=$1
+newerCommit=$2
+
+if [ -n "$3" ]; then
+ workingDir=$3
+ if [ -d "$workingDir" ]; then
+ echo "Directory ${workingDir} already exists. Use a new directory."
+ exit 1
+ fi
+else
+ # use the temp directory in case workingDir is not provided
+ workingDir=$TMP_DIR
+fi
+
+# create subdirectories for given commits
+oldTargetDir="$workingDir"/oldTargetDir
+newTargetDir="$workingDir"/newTargetDir
+
+if ! mkdir -p "$oldTargetDir"; then
+ echo "Failed to create target directory ${oldTargetDir}"
+ exit 1
+fi
+if ! mkdir -p "$newTargetDir"; then
+ echo "Failed to create target directory ${newTargetDir}"
+ exit 1
+fi
+
+# Building targets
+echo "Building the old version ... "
+checkoutAndBuild "$olderCommit" "$oldTargetDir"
+echo "Building the new version ..."
+checkoutAndBuild "$newerCommit" "$newTargetDir"
+
+# check that the default ports are open
+if [ "$(lsof -t -i:8097 -s TCP:LISTEN)" ] || [ "$(lsof -t -i:8098
-sTCP:LISTEN)" ] || [ "$(lsof -t -i:8099 -sTCP:LISTEN)" ] ||
+ [ "$(lsof -t -i:9000 -sTCP:LISTEN)" ] || [ "$(lsof -t -i:2181
-sTCP:LISTEN)" ]; then
+ echo "Cannot start the components since the default ports are not open.
Check any existing process that may be using the default ports."
+ exit 1
+fi
+
+# Setup initial cluster with olderCommit and do rolling upgrade
+startServices "$oldTargetDir"
+sleep 20
+stopService controller "$oldTargetDir"
+startService controller "$newTargetDir"
+stopService broker "$oldTargetDir"
+startService broker "$newTargetDir"
+stopService server "$oldTargetDir"
+startService server "$newTargetDir"
+sleep 20
+stopService controller "$newTargetDir"
+startService controller "$oldTargetDir"
+stopService broker "$newTargetDir"
+startService broker "$oldTargetDir"
+stopService server "$newTargetDir"
+startService server "$oldTargetDir"
+sleep 20
+stopServices "$oldTargetDir"
+
+exit 0
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
