Repository: incubator-edgent Updated Branches: refs/heads/develop fa6ade42e -> 68d0e7fe7
add release validation check_jars.sh Initial version of tools to help check that a release contains the appropriate jars and they have the required metadata content. This version lacks integration with Nexus - the real target of this validation checking. If there's a tool that can download all of the jars present in a nexus staging area then this tool can be run against that downloaded tree. Project: http://git-wip-us.apache.org/repos/asf/incubator-edgent/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-edgent/commit/68d0e7fe Tree: http://git-wip-us.apache.org/repos/asf/incubator-edgent/tree/68d0e7fe Diff: http://git-wip-us.apache.org/repos/asf/incubator-edgent/diff/68d0e7fe Branch: refs/heads/develop Commit: 68d0e7fe77969e33c11fe35a485f67b60990d5b7 Parents: fa6ade4 Author: Dale LaBossiere <dlab...@us.ibm.com> Authored: Fri Dec 15 14:23:31 2017 -0500 Committer: Dale LaBossiere <dlab...@us.ibm.com> Committed: Fri Dec 15 14:23:31 2017 -0500 ---------------------------------------------------------------------- buildTools/check_jar.sh | 80 ++++++++++++ buildTools/check_jars.sh | 215 +++++++++++++++++++++++++++++++ buildTools/release_jars_android.txt | 65 ++++++++++ buildTools/release_jars_std.txt | 64 +++++++++ 4 files changed, 424 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-edgent/blob/68d0e7fe/buildTools/check_jar.sh ---------------------------------------------------------------------- diff --git a/buildTools/check_jar.sh b/buildTools/check_jar.sh new file mode 100755 index 0000000..bdf2b23 --- /dev/null +++ b/buildTools/check_jar.sh @@ -0,0 +1,80 @@ +#!/bin/sh + +################################################################################ +## +## Licensed to the Apache Software Foundation (ASF) under one or more +## contributor license agreements. See the NOTICE file distributed with +## this work for additional information regarding copyright ownership. +## The ASF licenses this file to You under the Apache License, Version 2.0 +## (the "License"); you may not use this file except in compliance with +## the License. You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +################################################################################ + +set -e + +# Checks the binary release artifact (jar/war) contain the appropriate +# metadata (LICENSE,NOTICE,DISCLAIMER,DEPENDENCIES) + +. `dirname $0`/common.sh + +setUsage "`basename $0` [-v] jar-or-war-path" +handleHelp "$@" + +VERBOSE=0 +if [ "$1" == "-v" ] +then + VERBOSE=1; shift +fi + +requireArg "$@" +FILE=$1; shift +[ -f ${FILE} ] || die "File \"${FILE}\" does not exist" + +noExtraArgs "$@" + +function checkExists() { # $1=base $2=path + if [ ! -f "$1/$2" ] ; then + echo "Error: No such file $2" + return 1 + fi +} + +function checkJar() { # $1 abs-jar-pname + JAR="$1" + set +e + DIR=`mktemp -d` + mkdir ${DIR}/contents + EC=0 + [ ${EC} = 0 ] && (cd ${DIR}/contents; jar xf ${JAR}) || EC=1 + BASE=${DIR}/contents + if [ ${EC} = 0 ] ; then + checkExists ${BASE} META-INF/LICENSE || EC=1 + checkExists ${BASE} META-INF/NOTICE || EC=1 + checkExists ${BASE} META-INF/DISCLAIMER || EC=1 + + # ASF policy doesn't require DEPENDENCIES but our project policy does + checkExists ${BASE} META-INF/DEPENDENCIES || EC=1 + fi + (cd ${DIR}; rm -rf contents) + rmdir ${DIR} + set -e + if [ ${EC} != 0 ] ; then + echo "FAILED: Jar has incorrect metadata" + fi + return ${EC} +} + +ABS_FILE=$(getAbsPath ${FILE}) + +echo "Checking ${FILE} ..." + +checkJar ${ABS_FILE} # w/set -e, exits if returns non-0 http://git-wip-us.apache.org/repos/asf/incubator-edgent/blob/68d0e7fe/buildTools/check_jars.sh ---------------------------------------------------------------------- diff --git a/buildTools/check_jars.sh b/buildTools/check_jars.sh new file mode 100755 index 0000000..07bfa50 --- /dev/null +++ b/buildTools/check_jars.sh @@ -0,0 +1,215 @@ +#!/bin/sh + +################################################################################ +## +## Licensed to the Apache Software Foundation (ASF) under one or more +## contributor license agreements. See the NOTICE file distributed with +## this work for additional information regarding copyright ownership. +## The ASF licenses this file to You under the Apache License, Version 2.0 +## (the "License"); you may not use this file except in compliance with +## the License. You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +################################################################################ + +# Checks that binary release artifacts (jars/wars) contain the appropriate +# metadata (LICENSE,NOTICE,DISCLAIMER,DEPENDENCIES) +# and only the expected set of jars are present +# +# At the moment this tool only scans a source/built workspace for jars, +# and of course more jars than are released are present. +# That's OK as the tool still caught jar content problems in +# jars that are expected to be released. +# +# The script could also used against a get-edgent-jars.sh generated +# bundle (containing jars downloaded from nexus) which is also valuable, +# however it doesn't do anything to verify there aren't extra jars +# present in nexus (because get-edgent-jars only includes expected artifacts). +# To run against get-edgent-jars j8 (or j7,android) bundle: +# - extract get-edgent-jars bundle +# - buildTools/check_jars.sh --findmode nfilters --check j8 <ver> <extracted-dir>/libs +# It could be helpful in validating at least that content if we +# could for example point get-edgent-jars at a Nexus staged-release area. + +. `dirname $0`/common.sh + +setUsage "`basename $0` [--findmode {build|nfilters}] [--check {j8|j7|android},...] edgent-ver base-release-dir" +handleHelp "$@" + +FIND_MODE=build +if [ "$1" == "--findmode" -a $# -gt 1 ] ; then + FIND_MODE=$2; shift; shift +fi + +CHECK_CFG=j8,j7,android +if [ "$1" == "--check" -a $# -gt 1 ] ; then + CHECK_CFG=$2; shift; shift +fi + +requireArg "$@" +EDGENT_VER=$1; shift + +requireArg "$@" +BASE=$1; shift +[ -d ${BASE} ] || die "release-dir \"${BASE}\" does not exist" + +noExtraArgs "$@" + +BUILDTOOLS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +# get path of platform's expected jars list file +function getExpJarsPath() { # $1 = platform-id {"",std,j8,j7,android} + ID="$1" + [ "${ID}" == "" ] && ID="std" + FILE=${BUILDTOOLS_DIR}/release_jars_${ID}.txt + + if [ ${ID} != "std" -a ! -f ${FILE} ] ; then + ID="std" + FILE=${BUILDTOOLS_DIR}/release_jars_${ID}.txt + fi + + [ ! -f ${FILE} ] && die "No such release jars list file: ${FILE}" + + echo ${FILE} +} + +# get list of platform's expected jars leafnames +function getExpJarsList() { # $1 platform-id {j8,j7,android} $2 VER + ID="$1" + VER="$2" + FILE=`getExpJarsPath ${ID}` + # handle leading whitespace, ws-only lines, comment lines, trailing comments + # EXP_JARS=`cat ${FILE} | sed -e '/^[ \t]*$/d' -e '/^[ \t]*#/d' ` + EXP_JARS=`cat ${FILE} | sed -e 's/^[ \t]*//' -e '/^#/d' -e 's/[ \t]*#.*$//' -e 's/[ \t]*$//' ` + EXP_JARS=`for i in ${EXP_JARS} ; do echo ${i} | sed -e "s/{VER}/${VER}/g" ; done` + echo "${EXP_JARS}" +} + +# function to find actual jars/wars present in a workspace build +function findBuildJars() { # $1 BASE-DIR + BASE_DIR=$1 + + # ACTUAL - when scanning built source tree's "target" dirs, + # need to exclude those + # embedded in war (under WEB-INF) + # test classes jars + # under component's target test-resources or classes (e.g., a war) + # test components + # connectors-websocket-server (test) component + # for J8, those under platforms + # + ACTUAL="`find ${BASE_DIR} -name \*${EDGENT_VER}*.[jw]ar \ + | grep /target/ \ + | grep -v /WEB-INF/ \ + | grep -v -- -tests.jar \ + | grep -v /test-resources/ \ + | grep -v /classes/ \ + | grep -v /retrolambda/ \ + | grep -v '/test/.*/target/' \ + | grep -v '/edgent-connectors-websocket-server-' \ + `" + if [ "`echo ${BASE_DIR} | grep platforms`" = "" ] ; then + ACTUAL="`for i in ${ACTUAL} ; do echo ${i} | grep -v /platforms/ ; done`" + fi + + [ "${ACTUAL}" == "" ] && echo "WARNING: no files found under '${BASE_DIR}' for edgent version ${EDGENT_VER} " >/dev/stderr + + echo "${ACTUAL}" +} + +function findJarsNoFilter() { # $1 BASE-DIR + BASE_DIR=$1 + + ACTUAL="`find ${BASE_DIR} -name \*.[jw]ar `" + + [ "${ACTUAL}" == "" ] && echo "WARNING: no files found under '${BASE_DIR}' " >/dev/stderr + + echo "${ACTUAL}" +} + +function findJars() { # $1 mode:{build,noFilter} $2 BASE-DIR + MODE="$1" + BASE_DIR=$2 + if [ "build" == ${MODE} ] ; then + findBuildJars "${BASE_DIR}" + else + findJarsNoFilter "${BASE_DIR}" + fi +} + +function checkJars() { # $1 EXP-JARS $2 ACTUAL-JAR-PATHS + EXPECT="$1" + ACTUAL="$2" + + FEC=0 + echo "##### Checking Jar contents ..." + for i in ${ACTUAL} ; do + ${BUILDTOOLS_DIR}/check_jar.sh ${i} || FEC=1 + done + echo "##### done" + + # check matching lists + echo "##### Checking correct Jars are present ..." + ACTUAL="`for i in ${ACTUAL} ; do echo $(basename ${i}) ; done | sort`" # get basename + EXPECT="`for i in ${EXPECT} ; do echo "${i}" ; done | sort`" + + ACT_FILE=/tmp/$$-ACTUAL + for i in ${ACTUAL} ; do echo $i >> ${ACT_FILE} ; done + EXP_FILE=/tmp/$$-EXPECT + for i in ${EXPECT} ; do echo $i >> ${EXP_FILE} ; done + + [ "${ACTUAL}" = "${EXPECT}" ] || FEC=1 + (set -x; comm -3 ${ACT_FILE} ${EXP_FILE} ) + + rm ${ACT_FILE} ${EXP_FILE} + echo "##### done" + + if [ ${FEC} = 0 ] ; then + echo "##### Checking Jars OK" + else + echo "##### Checking Jars FAILED" + fi + + return ${FEC} +} + +EC=0 + +if [ "" != "$(echo $CHECK_CFG | grep j8)" ] ; then + echo + echo "##### Checking J8 jars ..." + ACTUAL=`findJars ${FIND_MODE} ${BASE}` + EXPECT=`getExpJarsList j8 ${EDGENT_VER}` + checkJars "${EXPECT}" "${ACTUAL}" || EC=1 +fi + +if [ "" != "$(echo $CHECK_CFG | grep j7)" ] ; then + echo + echo "##### Checking J7 jars ..." + ACTUAL=`findJars ${FIND_MODE} ${BASE}/platforms/java7` + EXPECT=`getExpJarsList j7 ${EDGENT_VER}` + checkJars "${EXPECT}" "${ACTUAL}" || EC=1 +fi + +if [ "" != "$(echo $CHECK_CFG | grep android)" ] ; then + echo + echo "##### Checking Android jars ..." + ACTUAL=`findJars ${FIND_MODE} ${BASE}/platforms/android` + EXPECT=`getExpJarsList android ${EDGENT_VER}` + checkJars "${EXPECT}" "${ACTUAL}" || EC=1 +fi + +echo +if [ ${EC} = 0 ] ; then + echo "##### Checking all platform Jars OK" +else + echo "##### Checking all platform Jars FAILED" + exit 1 +fi http://git-wip-us.apache.org/repos/asf/incubator-edgent/blob/68d0e7fe/buildTools/release_jars_android.txt ---------------------------------------------------------------------- diff --git a/buildTools/release_jars_android.txt b/buildTools/release_jars_android.txt new file mode 100644 index 0000000..85bcf39 --- /dev/null +++ b/buildTools/release_jars_android.txt @@ -0,0 +1,65 @@ +################################################################################ +## +## Licensed to the Apache Software Foundation (ASF) under one or more +## contributor license agreements. See the NOTICE file distributed with +## this work for additional information regarding copyright ownership. +## The ASF licenses this file to You under the Apache License, Version 2.0 +## (the "License"); you may not use this file except in compliance with +## the License. You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +################################################################################ + +# List of jars/wars that are to be included in an Android release. +# Comments, blank lines, and whitespace are allowed in all the usual places. +# This is used by some release validation tools (e.g., check_jars.sh) +# +edgent-android-hardware-{VER}.jar +edgent-android-topology-{VER}.jar +edgent-analytics-math3-{VER}.jar +edgent-analytics-sensors-{VER}.jar +edgent-api-execution-{VER}.jar +edgent-api-function-{VER}.jar +edgent-api-graph-{VER}.jar +edgent-api-oplet-{VER}.jar +edgent-api-topology-{VER}.jar +edgent-api-window-{VER}.jar +edgent-apps-iot-{VER}.jar +edgent-apps-runtime-{VER}.jar +edgent-connectors-command-{VER}.jar +edgent-connectors-common-{VER}.jar +edgent-connectors-csv-{VER}.jar +edgent-connectors-file-{VER}.jar +edgent-connectors-http-{VER}.jar +edgent-connectors-iot-{VER}.jar +edgent-connectors-iotp-{VER}.jar +edgent-connectors-jdbc-{VER}.jar +edgent-connectors-kafka-{VER}.jar +edgent-connectors-mqtt-{VER}.jar +edgent-connectors-pubsub-{VER}.jar +edgent-connectors-serial-{VER}.jar +edgent-connectors-websocket-{VER}.jar +edgent-connectors-websocket-base-{VER}.jar +edgent-connectors-websocket-jetty-{VER}.jar +edgent-connectors-websocket-misc-{VER}.jar +# edgent-console-server-{VER}.jar +# edgent-console-servlets-{VER}.war +# edgent-providers-development-{VER}.jar +edgent-providers-direct-{VER}.jar +edgent-providers-iot-{VER}.jar +edgent-runtime-appservice-{VER}.jar +edgent-runtime-etiao-{VER}.jar +# edgent-runtime-jmxcontrol-{VER}.jar +edgent-runtime-jobregistry-{VER}.jar +edgent-runtime-jsoncontrol-{VER}.jar +edgent-spi-graph-{VER}.jar +edgent-spi-topology-{VER}.jar +edgent-utils-metrics-{VER}.jar +edgent-utils-streamscope-{VER}.jar http://git-wip-us.apache.org/repos/asf/incubator-edgent/blob/68d0e7fe/buildTools/release_jars_std.txt ---------------------------------------------------------------------- diff --git a/buildTools/release_jars_std.txt b/buildTools/release_jars_std.txt new file mode 100644 index 0000000..0c96e18 --- /dev/null +++ b/buildTools/release_jars_std.txt @@ -0,0 +1,64 @@ +################################################################################ +## +## Licensed to the Apache Software Foundation (ASF) under one or more +## contributor license agreements. See the NOTICE file distributed with +## this work for additional information regarding copyright ownership. +## The ASF licenses this file to You under the Apache License, Version 2.0 +## (the "License"); you may not use this file except in compliance with +## the License. You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +################################################################################ + +# List of jars/wars that are to be included in a "standard" (Java8, Java7) +# release. +# Comments, blank lines, and whitespace are allowed in all the usual places. +# This is used by some release validation tools (e.g., check_jars.sh) +# +edgent-analytics-math3-{VER}.jar +edgent-analytics-sensors-{VER}.jar +edgent-api-execution-{VER}.jar +edgent-api-function-{VER}.jar +edgent-api-graph-{VER}.jar +edgent-api-oplet-{VER}.jar +edgent-api-topology-{VER}.jar +edgent-api-window-{VER}.jar +edgent-apps-iot-{VER}.jar +edgent-apps-runtime-{VER}.jar +edgent-connectors-command-{VER}.jar +edgent-connectors-common-{VER}.jar +edgent-connectors-csv-{VER}.jar +edgent-connectors-file-{VER}.jar +edgent-connectors-http-{VER}.jar +edgent-connectors-iot-{VER}.jar +edgent-connectors-iotp-{VER}.jar +edgent-connectors-jdbc-{VER}.jar +edgent-connectors-kafka-{VER}.jar +edgent-connectors-mqtt-{VER}.jar +edgent-connectors-pubsub-{VER}.jar +edgent-connectors-serial-{VER}.jar +edgent-connectors-websocket-{VER}.jar +edgent-connectors-websocket-base-{VER}.jar +edgent-connectors-websocket-jetty-{VER}.jar +edgent-connectors-websocket-misc-{VER}.jar +edgent-console-server-{VER}.jar +edgent-console-servlets-{VER}.war +edgent-providers-development-{VER}.jar +edgent-providers-direct-{VER}.jar +edgent-providers-iot-{VER}.jar +edgent-runtime-appservice-{VER}.jar +edgent-runtime-etiao-{VER}.jar +edgent-runtime-jmxcontrol-{VER}.jar +edgent-runtime-jobregistry-{VER}.jar +edgent-runtime-jsoncontrol-{VER}.jar +edgent-spi-graph-{VER}.jar +edgent-spi-topology-{VER}.jar +edgent-utils-metrics-{VER}.jar +edgent-utils-streamscope-{VER}.jar
