# ignite-456: Implemented
Project: http://git-wip-us.apache.org/repos/asf/incubator-ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ignite/commit/3cc3fbe3 Tree: http://git-wip-us.apache.org/repos/asf/incubator-ignite/tree/3cc3fbe3 Diff: http://git-wip-us.apache.org/repos/asf/incubator-ignite/diff/3cc3fbe3 Branch: refs/heads/ignite-816 Commit: 3cc3fbe3168acaf49e691ffe74d780239968b311 Parents: c4d81fe Author: null <null> Authored: Wed May 20 15:45:33 2015 +0300 Committer: null <null> Committed: Wed May 20 15:45:33 2015 +0300 ---------------------------------------------------------------------- dev-tools/build.gradle | 35 +- dev-tools/gradle/wrapper/gradle-wrapper.jar | Bin 0 -> 51017 bytes .../gradle/wrapper/gradle-wrapper.properties | 6 + dev-tools/gradlew | 164 ++++++++++ dev-tools/src/main/groovy/jiraslurp.groovy | 326 ++++++++++++++----- parent/pom.xml | 4 + 6 files changed, 442 insertions(+), 93 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/3cc3fbe3/dev-tools/build.gradle ---------------------------------------------------------------------- diff --git a/dev-tools/build.gradle b/dev-tools/build.gradle index b760bc1..164a775 100644 --- a/dev-tools/build.gradle +++ b/dev-tools/build.gradle @@ -14,6 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + apply plugin: 'groovy' repositories { @@ -25,21 +26,29 @@ dependencies { } task help { - println '''There are two interfaces to work with JIRA attachment validation tool - - to do the batch validation of all latest patch attachments - gradle slurp - - to grab a single JIRA's latest attachment and run test validation on it - JIRA_NUM=INGITE-### gradle patchapply''' + println '''There are 3 interfaces to work with JIRA attachment validation tool + - to do the batch validation of all latest patch attachments + gradle slurp + - to grab a single JIRA's latest attachment and run test validation on it + JIRA_NUM=INGITE-### gradle patchapply + - to run all test builds for a single JIRA's latest attachment and run test validation on it + JIRA_NUM=INGITE-### gradle runAllBuilds''' +} + +task slurp(dependsOn: 'classes', type: JavaExec) { + args(project.buildDir, "slurp,${System.getenv('TEST_BUILDS')}") + main = 'jiraslurp' + classpath = sourceSets.main.runtimeClasspath } -task slurp (dependsOn: 'classes', type: JavaExec) { - args (project.buildDir, 'slurp') - main = 'jiraslurp' - classpath = sourceSets.main.runtimeClasspath +task patchapply(dependsOn: 'classes', type: JavaExec) { + args("patchApply,${System.getenv('JIRA_NUM')}") + main = 'jiraslurp' + classpath = sourceSets.main.runtimeClasspath } -task patchapply (dependsOn: 'classes', type: JavaExec) { - args ("JIRA_NUM=${System.getenv('JIRA_NUM')}") - main = 'jiraslurp' - classpath = sourceSets.main.runtimeClasspath +task runAllBuilds(dependsOn: 'classes', type: JavaExec) { + args("runAllBuilds,${System.getenv('JIRA_NUM')}") + main = 'jiraslurp' + classpath = sourceSets.main.runtimeClasspath } http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/3cc3fbe3/dev-tools/gradle/wrapper/gradle-wrapper.jar ---------------------------------------------------------------------- diff --git a/dev-tools/gradle/wrapper/gradle-wrapper.jar b/dev-tools/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 0000000..b761216 Binary files /dev/null and b/dev-tools/gradle/wrapper/gradle-wrapper.jar differ http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/3cc3fbe3/dev-tools/gradle/wrapper/gradle-wrapper.properties ---------------------------------------------------------------------- diff --git a/dev-tools/gradle/wrapper/gradle-wrapper.properties b/dev-tools/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 0000000..3111cd7 --- /dev/null +++ b/dev-tools/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,6 @@ +#Tue Feb 24 21:36:05 PST 2015 +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-2.0-bin.zip http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/3cc3fbe3/dev-tools/gradlew ---------------------------------------------------------------------- diff --git a/dev-tools/gradlew b/dev-tools/gradlew new file mode 100755 index 0000000..91a7e26 --- /dev/null +++ b/dev-tools/gradlew @@ -0,0 +1,164 @@ +#!/usr/bin/env bash + +############################################################################## +## +## Gradle start up script for UN*X +## +############################################################################## + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS="" + +APP_NAME="Gradle" +APP_BASE_NAME=`basename "$0"` + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD="maximum" + +warn ( ) { + echo "$*" +} + +die ( ) { + echo + echo "$*" + echo + exit 1 +} + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +case "`uname`" in + CYGWIN* ) + cygwin=true + ;; + Darwin* ) + darwin=true + ;; + MINGW* ) + msys=true + ;; +esac + +# For Cygwin, ensure paths are in UNIX format before anything is touched. +if $cygwin ; then + [ -n "$JAVA_HOME" ] && JAVA_HOME=`cygpath --unix "$JAVA_HOME"` +fi + +# Attempt to set APP_HOME +# Resolve links: $0 may be a link +PRG="$0" +# Need this for relative symlinks. +while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "$PRG"`"/$link" + fi +done +SAVED="`pwd`" +cd "`dirname \"$PRG\"`/" >&- +APP_HOME="`pwd -P`" +cd "$SAVED" >&- + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD="java" + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if [ "$cygwin" = "false" -a "$darwin" = "false" ] ; then + MAX_FD_LIMIT=`ulimit -H -n` + if [ $? -eq 0 ] ; then + if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then + MAX_FD="$MAX_FD_LIMIT" + fi + ulimit -n $MAX_FD + if [ $? -ne 0 ] ; then + warn "Could not set maximum file descriptor limit: $MAX_FD" + fi + else + warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" + fi +fi + +# For Darwin, add options to specify how the application appears in the dock +if $darwin; then + GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" +fi + +# For Cygwin, switch paths to Windows format before running java +if $cygwin ; then + APP_HOME=`cygpath --path --mixed "$APP_HOME"` + CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` + + # We build the pattern for arguments to be converted via cygpath + ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` + SEP="" + for dir in $ROOTDIRSRAW ; do + ROOTDIRS="$ROOTDIRS$SEP$dir" + SEP="|" + done + OURCYGPATTERN="(^($ROOTDIRS))" + # Add a user-defined pattern to the cygpath arguments + if [ "$GRADLE_CYGPATTERN" != "" ] ; then + OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" + fi + # Now convert the arguments - kludge to limit ourselves to /bin/sh + i=0 + for arg in "$@" ; do + CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` + CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option + + if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition + eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` + else + eval `echo args$i`="\"$arg\"" + fi + i=$((i+1)) + done + case $i in + (0) set -- ;; + (1) set -- "$args0" ;; + (2) set -- "$args0" "$args1" ;; + (3) set -- "$args0" "$args1" "$args2" ;; + (4) set -- "$args0" "$args1" "$args2" "$args3" ;; + (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; + (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; + (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; + (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; + (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; + esac +fi + +# Split up the JVM_OPTS And GRADLE_OPTS values into an array, following the shell quoting and substitution rules +function splitJvmOpts() { + JVM_OPTS=("$@") +} +eval splitJvmOpts $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS +JVM_OPTS[${#JVM_OPTS[*]}]="-Dorg.gradle.appname=$APP_BASE_NAME" + +exec "$JAVACMD" "${JVM_OPTS[@]}" -classpath "$CLASSPATH" org.gradle.wrapper.GradleWrapperMain "$@" http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/3cc3fbe3/dev-tools/src/main/groovy/jiraslurp.groovy ---------------------------------------------------------------------- diff --git a/dev-tools/src/main/groovy/jiraslurp.groovy b/dev-tools/src/main/groovy/jiraslurp.groovy index 0a876fa..34ac79c 100644 --- a/dev-tools/src/main/groovy/jiraslurp.groovy +++ b/dev-tools/src/main/groovy/jiraslurp.groovy @@ -20,33 +20,52 @@ */ final GIT_REPO = "https://git1-us-west.apache.org/repos/asf/incubator-ignite.git"; final ATTACHMENT_URL = "https://issues.apache.org/jira/secure/attachment"; -final validated_filename = "validated-jira.txt" -final LAST_SUCCESSFUL_ARTIFACT = "guestAuth/repository/download/Ignite_PatchValidation_Main/.lastSuccessful/$validated_filename" +final validated_filename = "${System.getProperty("user.home")}/validated-jira.txt" +final LAST_SUCCESSFUL_ARTIFACT = "guestAuth/repository/download/Ignite_PatchValidation_PatchChecker/.lastSuccessful/$validated_filename" final def JIRA_CMD = System.getProperty('JIRA_COMMAND', 'jira.sh') LinkedHashMap<String, String> jirasAttached = [:] +/** + * Gets jiras for which test tasks were already triggered. + * + * @return List of pairs [jira,attachemntId]. + */ def readHistory = { - final int MAX_HISTORY = 5000 - - List validated_list = [] - def validated = new File(validated_filename) - if (validated.exists()) { - validated_list = validated.text.split('\n') - validated.delete() - } else { - try { - validated_list = - new URL("http://204.14.53.152/$LAST_SUCCESSFUL_ARTIFACT";).text.split('\n') - } catch (Exception e) { - println e.getMessage() + final int MAX_HISTORY = 5000 + + List validated_list = [] + + // TODO do not use folder. + def validated = new File(validated_filename) + + if (validated.exists()) { + // TODO use commented way. + validated_list = validated.text.split('\n') } - } - // Let's make sure the preserved history isn't too long - if (validated_list.size > MAX_HISTORY) { - validated_list = validated_list[validated_list.size-MAX_HISTORY..validated_list.size-1] - } - validated_list + + // TODO use it way. +// try { +// def historyUrl = "http://${System.getenv('TC_URL')}/$LAST_SUCCESSFUL_ARTIFACT" +// +// println "Reading history from $historyUrl" +// +// validated_list = new URL(historyUrl).text.split('\n') +// +// println "Got validated list=$validated_list" +// } +// catch (Exception e) { +// println e.getMessage() +// +// } + + // Let's make sure the preserved history isn't too long + if (validated_list.size > MAX_HISTORY) + validated_list = validated_list[validated_list.size - MAX_HISTORY..validated_list.size - 1] + + println "History=$validated_list" + + validated_list } /** @@ -54,83 +73,230 @@ def readHistory = { * @return <code>null</code> or <code>JIRA-###,latest_attach_id</code> */ def getLatestAttachment = { jira -> - def latestAttr = jira.attachments[0].attachment.list().sort { - it.@id.toInteger() - }.reverse()[0] - String row = null - if (latestAttr == null) { - println "${jira.key} is in invalid state: patch is not available" - } else { - row = "${jira.key},${latestAttr.@id}" - } + def latestAttr = jira.attachments[0].attachment.list().sort { + it.@id.toInteger() + }.reverse()[0] + + String row = null + + if (latestAttr == null) { + println "${jira.key} is in invalid state: patch is not available" + } + else { + row = "${jira.key},${latestAttr.@id}" + } } def checkForAttachments = { - def JIRA_FILTER = - "https://issues.apache.org/jira/sr/jira.issueviews:searchrequest-xml/12330308/SearchRequest-12330308.xml?tempMax=100&field=key&field=attachments"; - def rss = new XmlSlurper().parse(JIRA_FILTER) - List list = readHistory{} - - rss.channel.item.each { jira -> - String row = getLatestAttachment (jira) - if (row != null && !list.contains(row)) { - def pair = row.split(',') - jirasAttached.put(pair[0] as String, pair[1] as String) - list.add(row) + def JIRA_FILTER = + "https://issues.apache.org/jira/sr/jira.issueviews:searchrequest-xml/12330308/SearchRequest-12330308.xml?tempMax=100&field=key&field=attachments"; + def rss = new XmlSlurper().parse(JIRA_FILTER) + + List list = readHistory {} + + rss.channel.item.each { jira -> + String row = getLatestAttachment(jira) + + if (row != null && !list.contains(row)) { + def pair = row.split(',') + + jirasAttached.put(pair[0] as String, pair[1] as String) + + list.add(row) + } } - } - // Write everything back to persist the list - def validated = new File(validated_filename) - validated << list.join('\n') + // Write everything back to persist the list + def validated = new File(validated_filename) + + if (validated.exists()) + validated.delete() + + validated << list.join('\n') } +/** + * Monitors given process and show errors if exist. + */ def checkprocess = { process -> - process.waitFor() - if (process.exitValue() != 0) { - println "Return code: " + process.exitValue() - println "Errout:\n" + process.err.text - assert process.exitValue() == 0 || process.exitValue() == 128 - } + println process.text + + process.waitFor() + + if (process.exitValue() != 0) { + println "Return code: " + process.exitValue() + println "Errout:\n" + process.err.text + + assert process.exitValue() == 0 || process.exitValue() == 128 + } } -def create_gitbranch = {jira, attachementURL -> - println jira - GIT_REPO - println "$ATTACHMENT_URL/$attachementURL/" - - def patchFile = new File("${jira}.patch") - patchFile << new URL("$ATTACHMENT_URL/$attachementURL/").text - checkprocess "git clone --depth 1 $GIT_REPO".execute() - checkprocess "git checkout -b sprint-2 origin/sprint-2".execute(null, new File('incubator-ignite')) - checkprocess "git am ../${patchFile.name}".execute(null, new File('incubator-ignite')) - patchFile.delete() +/** + * Applys patch from jira to given git state. + */ +def applyPatch = { jira, attachementURL -> + def userEmail = System.getenv("env.GIT_USER_EMAIL"); + def userName = System.getenv("env.GIT_USER_NAME"); + + println "Patch apllying with jira='$jira' and attachment='$ATTACHMENT_URL/$attachementURL/'." + + def patchFile = new File("${jira}-${attachementURL}.patch") + + try { + patchFile << new URL("$ATTACHMENT_URL/$attachementURL/").text + + try { + checkprocess "git branch".execute() + + checkprocess "git config user.email \"$userEmail\"".execute(null, new File("../")) + checkprocess "git config user.name \"$userName\"".execute(null, new File("../")) + + // Create a new uniqueue branch to applying patch + def newTestBranch = "test-branch-${jira}-${attachementURL}-${System.currentTimeMillis()}" + checkprocess "git checkout -b ${newTestBranch}".execute(null, new File("../")) + + checkprocess "git branch".execute() + + checkprocess "git am dev-tools/${patchFile.name}".execute(null, new File("../")) + + println "Patch was applied successfully." + } + catch (Exception e) { + println "Patch was not applied successfully. Aborting patch applying." + + checkprocess "git am --abort".execute(null, new File("../")) + + throw e; + } + } + finally { + assert patchFile.delete(), 'Could not delete patch file.' + } } def JIRA_xml = { jiranum -> - "https://issues.apache.org/jira/si/jira.issueviews:issue-xml/$jiranum/${jiranum}.xml"; + "https://issues.apache.org/jira/si/jira.issueviews:issue-xml/$jiranum/${jiranum}.xml"; +} + +/** + * Runs all given test builds to validate last patch from given jira. + */ +def runAllTestBuilds = { builds, jiraNum -> + assert jiraNum != 'null', 'Jira number should not be null.' + assert jiraNum != null, 'Jira number should not be null.' + + if (jiraNum) { + def tcURL = System.getenv('TC_URL') + def user = System.getenv('TASK_RUNNER_USER') + def pwd = System.getenv('TASK_RUNNER_PWD') + + builds.each { + try { + println "Triggering $it build for $jiraNum jira..." + + String postData = + "<build>" + + " <buildType id='$it'/>" + + " <properties>" + + " <property name='JIRA_NUM' value='$jiraNum'/>" + + " </properties>" + + "</build>"; + + URL url = new URL("http://$tcURL:80/httpAuth/app/rest/buildQueue";); + + HttpURLConnection conn = (HttpURLConnection)url.openConnection(); + + String encoded = new sun.misc.BASE64Encoder().encode("$user:$pwd".getBytes()); + + conn.setRequestProperty("Authorization", "Basic " + encoded); + + conn.setDoOutput(true); + conn.setRequestMethod("POST"); + conn.setRequestProperty("Content-Type", "application/xml"); + conn.setRequestProperty("Content-Length", String.valueOf(postData.length())); + + OutputStream os = conn.getOutputStream(); + os.write(postData.getBytes()); + os.flush(); + os.close(); + + conn.connect(); + + // Read response. + print "Response: " + + BufferedReader br = new BufferedReader(new InputStreamReader(conn.getInputStream())); + + String line; + while ((line = br.readLine()) != null) + println line + + br.close(); + } + catch (Exception e) { + e.printStackTrace() + } + } + } } +/** + * Main. + */ args.each { - println it - def parameters = it.split('=') - - if (parameters[0] == 'slurp') { - checkForAttachments() - // For each ticket with new attachment, let's trigger remove build - jirasAttached.each { k, v -> - // Trailing slash is important for download; only need to pass JIRA number - println "Triggering the build for: $k = $ATTACHMENT_URL/$v/" + println "Arg=$it" + + def parameters = it.split(",") + + println parameters + + if (parameters.length == 2 && parameters[0] == "slurp" && parameters[1] != 'null') { + def builds = parameters[1].split(' '); + + println "Running in 'slurp' mode. Test builds=${builds}" + + checkForAttachments() + + // For each ticket with new attachment, let's trigger remove build + jirasAttached.each { k, v -> + // Trailing slash is important for download; only need to pass JIRA number + println "Triggering the test builds for: $k = $ATTACHMENT_URL/$v/" + + runAllTestBuilds(builds,k) + } + } + else if (parameters.length == 2 && parameters[0] == "patchApply" && parameters[1] ==~ /\w+-\d+/) { + def jiraNum = parameters[1] + + println "Running in 'patch apply' mode with jira number '$jiraNum'" + + // Extract JIRA rss from the and pass the ticket element into attachment extraction + def rss = new XmlSlurper().parse(JIRA_xml(parameters[1])) + + String row = getLatestAttachment(rss.channel.item) + + println "Got row: $row." + + if (row != null) { + def pair = row.split(',') + def jira = pair[0] + def attachementURL = pair[1] + + applyPatch(jira, attachementURL) + } } - } else if (parameters.length == 2 && parameters[0] == 'JIRA_NUM' && parameters[1] ==~ /\w+-\d+/) { - // Extract JIRA rss from the and pass the ticket element into attachment extraction - def rss = new XmlSlurper().parse(JIRA_xml(parameters[1])) - String row = getLatestAttachment(rss.channel.item) - if (row != null) { - def pair = row.split(',') - create_gitbranch(pair[0], pair[1]) + else if (parameters.length >= 2 && parameters[0] == "runAllBuilds" && parameters[1] ==~ /\w+-\d+/) { + def jiraNum = parameters[1] + + def attachementURL=null + + if (parameters[2] ==~ /\d+/) + attachementURL = parameters[2] + + println "Running in 'all builds' mode with jira number='$jiraNum' and attachment URL='$attachementURL'." + + runAllTestBuilds jiraNum attachmentURL } - } } /* Workflow: http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/3cc3fbe3/parent/pom.xml ---------------------------------------------------------------------- diff --git a/parent/pom.xml b/parent/pom.xml index cd098d9..f06586a 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -634,6 +634,10 @@ <exclude>src/main/java/META-INF/services/javax.cache.spi.CachingProvider</exclude><!--cannot be changed--> <exclude>src/main/java/org/jetbrains/annotations/*.java</exclude><!--copyright--> <exclude>src/main/resources/META-INF/services/org.apache.hadoop.mapreduce.protocol.ClientProtocolProvider</exclude><!--cannot be changed--> + <exclude>dev-tools/IGNITE-*.patch</exclude> + <exclude>dev-tools/.gradle/**/*</exclude> + <exclude>dev-tools/gradle/wrapper/**/*</exclude> + <exclude>dev-tools/gradlew</exclude> <!--shmem--> <exclude>ipc/shmem/**/Makefile.in</exclude><!--auto generated files--> <exclude>ipc/shmem/**/Makefile</exclude><!--auto generated files-->
