Repository: kylin Updated Branches: refs/heads/master 82fc92f21 -> b3552c977
KYLIN-3008 add submit-patch.py Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b3552c97 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b3552c97 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b3552c97 Branch: refs/heads/master Commit: b3552c977a91af2824c7fcfe1c446b280e2940a8 Parents: 82fc92f Author: shaofengshi <[email protected]> Authored: Sun Nov 5 09:59:51 2017 +0800 Committer: shaofengshi <[email protected]> Committed: Sun Nov 5 15:07:36 2017 +0800 ---------------------------------------------------------------------- dev-support/make_patch.sh | 156 ++++++++++++++++ dev-support/python-requirements.txt | 21 +++ dev-support/submit-patch.py | 311 +++++++++++++++++++++++++++++++ 3 files changed, 488 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/b3552c97/dev-support/make_patch.sh ---------------------------------------------------------------------- diff --git a/dev-support/make_patch.sh b/dev-support/make_patch.sh new file mode 100755 index 0000000..8179098 --- /dev/null +++ b/dev-support/make_patch.sh @@ -0,0 +1,156 @@ +#!/bin/bash +#/** +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +# Make a patch for the current branch based on its tracking branch + +# Process args +while getopts "ahd:b:" opt; do + case "$opt" in + a) addendum='-addendum' + ;; + d) + patch_dir=$OPTARG + ;; + b) + tracking_branch=$OPTARG + ;; + *) + echo -e "Usage: $0 [-h] [-a] [-d] <directory> \n\ + Must be run from within the git branch to make the patch against.\n\ + -h - display these instructions.\n\ + -a - Add an 'addendum' prefix to the patch name.\n\ + -b - Specify the base branch to diff from. (defaults to the tracking branch or origin master)\n\ + -d - specify a patch directory (defaults to ~/patches/)" + exit 0 + ;; + esac +done + +# Find what branch we are on +branch=$(git branch |grep '*' |awk '{print $2}') +if [ ! "$branch" ]; then + echo "Can't determine the git branch. Exiting." >&2 + exit 1 +fi + +# Exit if git status is dirty +git_dirty=$(git diff --shortstat 2> /dev/null | wc -l|awk {'print $1'}) +echo "git_dirty is $git_dirty" +if [ "$git_dirty" -ne 0 ]; then + echo "Git status is dirty. Commit locally first.">&2 + exit 1 +fi + +# Determine the tracking branch if needed. +# If it was passed in from the command line +# with -b then use dthat no matter what. +if [ ! "$tracking_branch" ]; then + git log -n 1 origin/$branch > /dev/null 2>&1 + status=$? + if [ "$status" -eq 128 ]; then + # Status 128 means there is no remote branch + tracking_branch='origin/master' + elif [ "$status" -eq 0 ]; then + # Status 0 means there is a remote branch + tracking_branch="origin/$branch" + else + echo "Unknown error: $?" >&2 + exit 1 + fi +fi + + +# Deal with invalid or missing $patch_dir +if [ ! "$patch_dir" ]; then + echo -e "Patch directory not specified. Falling back to ~/patches/." + patch_dir=~/patches +fi + +if [ ! -d "$patch_dir" ]; then + echo "$patch_dir does not exist. Creating it." + mkdir $patch_dir +fi + +# Determine what to call the patch +# Check to see if any patch exists that includes the branch name +status=$(ls $patch_dir/*$branch* 2>/dev/null|grep -v addendum|wc -l|awk {'print $1'}) +if [ "$status" -eq 0 ]; then + # This is the first patch we are making for this release + prefix='' +elif [ "$status" -ge 1 ]; then + # At least one patch already exists -- add a version prefix + for i in {1..99}; do + # Check to see the maximum version of patch that exists + if [ ! -f "$patch_dir/$branch.v$i.patch" ]; then + version=$i + if [ -n "$addendum" ]; then + # Don't increment the patch # if it is an addendum + echo "Creating an addendum" + if [ "$version" -eq 1 ]; then + # We are creating an addendum to the first version of the patch + prefix='' + else + # We are making an addendum to a different version of the patch + let version=$version-1 + prefix=".v$version" + fi + else + prefix=".v$version" + fi + break + fi + done +fi +# If this is against a tracking branch other than master +# include it in the patch name +tracking_suffix="" +if [[ $tracking_branch != "origin/master" \ + && $tracking_branch != "master" ]]; then + tracking_suffix=".${tracking_branch#origin/}" +fi + +patch_name="$branch$prefix$addendum$tracking_suffix.patch" + +# Do we need to make a diff? +git diff --quiet $tracking_branch +status=$? +if [ "$status" -eq 0 ]; then + echo "There is no difference between $branch and $tracking_branch." + echo "No patch created." + exit 0 +fi + +# Check whether we need to squash or not +local_commits=$(git log $tracking_branch..$branch|grep 'Author:'|wc -l|awk {'print $1'}) +if [ "$local_commits" -gt 1 ]; then + read -p "$local_commits commits exist only in your local branch. Interactive rebase?" yn + case $yn in + [Yy]* ) + git rebase -i $tracking_branch + ;; + [Nn]* ) + echo "Creating $patch_dir/$patch_name using git diff." + git diff $tracking_branch > $patch_dir/$patch_name + exit 0 + ;; + esac +fi + +echo "Creating patch $patch_dir/$patch_name using git format-patch" +git format-patch --stdout $tracking_branch > $patch_dir/$patch_name http://git-wip-us.apache.org/repos/asf/kylin/blob/b3552c97/dev-support/python-requirements.txt ---------------------------------------------------------------------- diff --git a/dev-support/python-requirements.txt b/dev-support/python-requirements.txt new file mode 100644 index 0000000..e7fcf31 --- /dev/null +++ b/dev-support/python-requirements.txt @@ -0,0 +1,21 @@ +## +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +requests +gitpython +rbtools +jinja2 http://git-wip-us.apache.org/repos/asf/kylin/blob/b3552c97/dev-support/submit-patch.py ---------------------------------------------------------------------- diff --git a/dev-support/submit-patch.py b/dev-support/submit-patch.py new file mode 100755 index 0000000..babf869 --- /dev/null +++ b/dev-support/submit-patch.py @@ -0,0 +1,311 @@ +#!/usr/bin/env python +## +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Makes a patch for the current branch, creates/updates the review board request and uploads new +# patch to jira. Patch is named as (JIRA).(branch name).(patch number).patch as per Yetus' naming +# rules. If no jira is specified, patch will be named (branch name).patch and jira and review board +# are not updated. Review board id is retrieved from the remote link in the jira. +# Print help: submit-patch.py --h +import argparse +import getpass +import git +import json +import logging +import os +import re +import requests +import subprocess +import sys + +parser = argparse.ArgumentParser( + epilog = "To avoid having to enter jira/review board username/password every time, setup an " + "encrypted ~/.apache-cred files as follows:\n" + "1) Create a file with following single " + "line: \n{\"jira_username\" : \"appy\", \"jira_password\":\"123\", " + "\"rb_username\":\"appy\", \"rb_password\" : \"@#$\"}\n" + "2) Encrypt it with openssl.\n" + "openssl enc -aes-256-cbc -in <file> -out ~/.apache-creds\n" + "3) Delete original file.\n" + "Now onwards, you'll need to enter this encryption key only once per run. If you " + "forget the key, simply regenerate ~/.apache-cred file again.", + formatter_class=argparse.RawTextHelpFormatter +) +parser.add_argument("-b", "--branch", + help = "Branch to use for generating diff. If not specified, tracking branch " + "is used. If there is no tracking branch, error will be thrown.") + +# Arguments related to Jira. +parser.add_argument("-jid", "--jira-id", + help = "Jira id of the issue. If set, we deduce next patch version from " + "attachments in the jira and also upload the new patch. Script will " + "ask for jira username/password for authentication. If not set, " + "patch is named <branch>.patch.") + +# Arguments related to Review Board. +parser.add_argument("-srb", "--skip-review-board", + help = "Don't create/update the review board.", + default = False, action = "store_true") +parser.add_argument("--reviewers", + help = "Comma separated list of users to add as reviewers.") + +# Misc arguments +parser.add_argument("--patch-dir", default = "~/patches", + help = "Directory to store patch files. If it doesn't exist, it will be " + "created. Default: ~/patches") +parser.add_argument("--rb-repo", default = "kylin-git", + help = "Review board repository. Default: kylin-git") +args = parser.parse_args() + +# Setup logger +logging.basicConfig() +logger = logging.getLogger("submit-patch") +logger.setLevel(logging.INFO) + + +def log_fatal_and_exit(*arg): + logger.fatal(*arg) + sys.exit(1) + + +def assert_status_code(response, expected_status_code, description): + if response.status_code != expected_status_code: + log_fatal_and_exit(" Oops, something went wrong when %s. \nResponse: %s %s\nExiting..", + description, response.status_code, response.reason) + + +# Make repo instance to interact with git repo. +try: + repo = git.Repo(os.getcwd()) + git = repo.git +except git.exc.InvalidGitRepositoryError as e: + log_fatal_and_exit(" '%s' is not valid git repo directory.\nRun from base directory of " + "HBase's git repo.", e) + +logger.info(" Active branch: %s", repo.active_branch.name) +# Do not proceed if there are uncommitted changes. +if repo.is_dirty(): + log_fatal_and_exit(" Git status is dirty. Commit locally first.") + + +# Returns base branch for creating diff. +def get_base_branch(): + # if --branch is set, use it as base branch for computing diff. Also check that it's a valid branch. + if args.branch is not None: + base_branch = args.branch + # Check that given branch exists. + for ref in repo.refs: + if ref.name == base_branch: + return base_branch + log_fatal_and_exit(" Branch '%s' does not exist in refs.", base_branch) + else: + # if --branch is not set, use tracking branch as base branch for computing diff. + # If there is no tracking branch, log error and quit. + tracking_branch = repo.active_branch.tracking_branch() + if tracking_branch is None: + log_fatal_and_exit(" Active branch doesn't have a tracking_branch. Please specify base " + " branch for computing diff using --branch flag.") + logger.info(" Using tracking branch as base branch") + return tracking_branch.name + + +# Returns patch name having format (JIRA).(branch name).(patch number).patch. If no jira is +# specified, patch is name (branch name).patch. +def get_patch_name(branch): + if args.jira_id is None: + return branch + ".patch" + + patch_name_prefix = args.jira_id.upper() + "." + branch + return get_patch_name_with_version(patch_name_prefix) + + +# Fetches list of attachments from the jira, deduces next version for the patch and returns final +# patch name. +def get_patch_name_with_version(patch_name_prefix): + # JIRA's rest api is broken wrt to attachments. https://jira.atlassian.com/browse/JRA-27637. + # Using crude way to get list of attachments. + url = "https://issues.apache.org/jira/browse/"; + args.jira_id + logger.info("Getting list of attachments for jira %s from %s", args.jira_id, url) + html = requests.get(url) + if html.status_code == 404: + log_fatal_and_exit(" Invalid jira id : %s", args.jira_id) + if html.status_code != 200: + log_fatal_and_exit(" Cannot fetch jira information. Status code %s", html.status_code) + # Iterate over patch names starting from version 1 and return when name is not already used. + content = unicode(html.content, 'utf-8') + for i in range(1, 1000): + name = patch_name_prefix + "." + ('{0:03d}'.format(i)) + ".patch" + if name not in content: + return name + + +# Validates that patch directory exists, if not, creates it. +def validate_patch_dir(patch_dir): + # Create patch_dir if it doesn't exist. + if not os.path.exists(patch_dir): + logger.warn(" Patch directory doesn't exist. Creating it.") + os.mkdir(patch_dir) + else: + # If patch_dir exists, make sure it's a directory. + if not os.path.isdir(patch_dir): + log_fatal_and_exit(" '%s' exists but is not a directory. Specify another directory.", + patch_dir) + + +# Make sure current branch is ahead of base_branch by exactly 1 commit. Quits if +# - base_branch has commits not in current branch +# - current branch is same as base branch +# - current branch is ahead of base_branch by more than 1 commits +def check_diff_between_branches(base_branch): + only_in_base_branch = git.log("HEAD.." + base_branch, oneline = True) + only_in_active_branch = git.log(base_branch + "..HEAD", oneline = True) + if len(only_in_base_branch) != 0: + log_fatal_and_exit(" '%s' is ahead of current branch by %s commits. Rebase " + "and try again.", base_branch, len(only_in_base_branch.split("\n"))) + if len(only_in_active_branch) == 0: + log_fatal_and_exit(" Current branch is same as '%s'. Exiting...", base_branch) + if len(only_in_active_branch.split("\n")) > 1: + log_fatal_and_exit(" Current branch is ahead of '%s' by %s commits. Squash into single " + "commit and try again.", + base_branch, len(only_in_active_branch.split("\n"))) + + +# If ~/.apache-creds is present, load credentials from it otherwise prompt user. +def get_credentials(): + creds = dict() + creds_filepath = os.path.expanduser("~/.apache-creds") + if os.path.exists(creds_filepath): + try: + logger.info(" Reading ~/.apache-creds for Jira and ReviewBoard credentials") + content = subprocess.check_output("openssl enc -aes-256-cbc -d -in " + creds_filepath, + shell=True) + except subprocess.CalledProcessError as e: + log_fatal_and_exit(" Couldn't decrypt ~/.apache-creds file. Exiting..") + creds = json.loads(content) + else: + creds['jira_username'] = raw_input("Jira username:") + creds['jira_password'] = getpass.getpass("Jira password:") + if not args.skip_review_board: + creds['rb_username'] = raw_input("Review Board username:") + creds['rb_password'] = getpass.getpass("Review Board password:") + return creds + + +def attach_patch_to_jira(issue_url, patch_filepath, creds): + # Upload patch to jira using REST API. + headers = {'X-Atlassian-Token': 'no-check'} + files = {'file': open(patch_filepath, 'rb')} + jira_auth = requests.auth.HTTPBasicAuth(creds['jira_username'], creds['jira_password']) + attachment_url = issue_url + "/attachments" + r = requests.post(attachment_url, headers = headers, files = files, auth = jira_auth) + assert_status_code(r, 200, "uploading patch to jira") + + +def get_jira_summary(issue_url): + r = requests.get(issue_url + "?fields=summary") + assert_status_code(r, 200, "fetching jira summary") + return json.loads(r.content)["fields"]["summary"] + + +def get_review_board_id_if_present(issue_url, rb_link_title): + r = requests.get(issue_url + "/remotelink") + assert_status_code(r, 200, "fetching remote links") + links = json.loads(r.content) + for link in links: + if link["object"]["title"] == rb_link_title: + res = re.search("reviews.apache.org/r/([0-9]+)", link["object"]["url"]) + return res.group(1) + return None + + +base_branch = get_base_branch() +# Remove remote repo name from branch name if present. This assumes that we don't use '/' in +# actual branch names. +base_branch_without_remote = base_branch.split('/')[-1] +logger.info(" Base branch: %s", base_branch) + +check_diff_between_branches(base_branch) + +patch_dir = os.path.abspath(os.path.expanduser(args.patch_dir)) +logger.info(" Patch directory: %s", patch_dir) +validate_patch_dir(patch_dir) + +patch_filename = get_patch_name(base_branch_without_remote) +logger.info(" Patch name: %s", patch_filename) +patch_filepath = os.path.join(patch_dir, patch_filename) + +diff = git.format_patch(base_branch, stdout = True) +with open(patch_filepath, "w") as f: + f.write(diff.encode('utf8')) + +if args.jira_id is not None: + creds = get_credentials() + issue_url = "https://issues.apache.org/jira/rest/api/2/issue/"; + args.jira_id + + attach_patch_to_jira(issue_url, patch_filepath, creds) + + if not args.skip_review_board: + rb_auth = requests.auth.HTTPBasicAuth(creds['rb_username'], creds['rb_password']) + + rb_link_title = "Review Board (" + base_branch_without_remote + ")" + rb_id = get_review_board_id_if_present(issue_url, rb_link_title) + + # If no review board link found, create new review request and add its link to jira. + if rb_id is None: + reviews_url = "https://reviews.apache.org/api/review-requests/"; + data = {"repository" : "kylin-git"} + r = requests.post(reviews_url, data = data, auth = rb_auth) + assert_status_code(r, 201, "creating new review request") + review_request = json.loads(r.content)["review_request"] + absolute_url = review_request["absolute_url"] + logger.info(" Created new review request: %s", absolute_url) + + # Use jira summary as review's summary too. + summary = get_jira_summary(issue_url) + # Use commit message as description. + description = git.log("-1", pretty="%B") + update_draft_data = {"bugs_closed" : [args.jira_id.upper()], "target_groups" : "kylin", + "target_people" : args.reviewers, "summary" : summary, + "description" : description } + draft_url = review_request["links"]["draft"]["href"] + r = requests.put(draft_url, data = update_draft_data, auth = rb_auth) + assert_status_code(r, 200, "updating review draft") + + draft_request = json.loads(r.content)["draft"] + diff_url = draft_request["links"]["draft_diffs"]["href"] + files = {'path' : (patch_filename, open(patch_filepath, 'rb'))} + r = requests.post(diff_url, files = files, auth = rb_auth) + assert_status_code(r, 201, "uploading diff to review draft") + + r = requests.put(draft_url, data = {"public" : True}, auth = rb_auth) + assert_status_code(r, 200, "publishing review request") + + # Add link to review board in the jira. + remote_link = json.dumps({'object': {'url': absolute_url, 'title': rb_link_title}}) + jira_auth = requests.auth.HTTPBasicAuth(creds['jira_username'], creds['jira_password']) + r = requests.post(issue_url + "/remotelink", data = remote_link, auth = jira_auth, + headers={'Content-Type':'application/json'}) + else: + logger.info(" Updating existing review board: https://reviews.apache.org/r/%s";, rb_id) + draft_url = "https://reviews.apache.org/api/review-requests/"; + rb_id + "/draft/" + diff_url = draft_url + "diffs/" + files = {'path' : (patch_filename, open(patch_filepath, 'rb'))} + r = requests.post(diff_url, files = files, auth = rb_auth) + assert_status_code(r, 201, "uploading diff to review draft") + + r = requests.put(draft_url, data = {"public" : True}, auth = rb_auth) + assert_status_code(r, 200, "publishing review request")
