hbase/dev-support/submit-patch.py

#!/usr/bin/env python2
##
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Makes a patch for the current branch, creates/updates the review board request and uploads new
# patch to jira. Patch is named as (JIRA).(branch name).(patch number).patch as per Yetus' naming
# rules. If no jira is specified, patch will be named (branch name).patch and jira and review board
# are not updated. Review board id is retrieved from the remote link in the jira.
# Print help: submit-patch.py --h
import argparse
from builtins import input, str
import getpass
import git
import json
import logging
import os
import re
import requests
import subprocess
import sys

parser = argparse.ArgumentParser(
    epilog = "To avoid having to enter jira/review board username/password every time, setup an "
             "encrypted ~/.apache-cred files as follows:\n"
             "1) Create a file with following single "
             "line: \n{\"jira_username\" : \"appy\", \"jira_password\":\"123\", "
             "\"rb_username\":\"appy\", \"rb_password\" : \"@#$\"}\n"
             "2) Encrypt it with openssl.\n"
             "openssl enc -aes-256-cbc -in <file> -out ~/.apache-creds\n"
             "3) Delete original file.\n"
             "Now onwards, you'll need to enter this encryption key only once per run. If you "
             "forget the key, simply regenerate ~/.apache-cred file again.",
    formatter_class=argparse.RawTextHelpFormatter
)
parser.add_argument("-b", "--branch",
                    help = "Branch to use for generating diff. If not specified, tracking branch "
                         "is used. If there is no tracking branch, error will be thrown.")

# Arguments related to Jira.
parser.add_argument("-jid", "--jira-id",
                    help = "Jira id of the issue. If set, we deduce next patch version from "
                           "attachments in the jira and also upload the new patch. Script will "
                           "ask for jira username/password for authentication. If not set, "
                           "patch is named <branch>.patch.")

# Arguments related to Review Board.
parser.add_argument("-srb", "--skip-review-board",
                    help = "Don't create/update the review board.",
                    default = False, action = "store_true")
parser.add_argument("--reviewers",
                    help = "Comma separated list of users to add as reviewers.")

# Misc arguments
parser.add_argument("--patch-dir", default = "~/patches",
                    help = "Directory to store patch files. If it doesn't exist, it will be "
                          "created. Default: ~/patches")
parser.add_argument("--rb-repo", default = "hbase-git",
                    help = "Review board repository. Default: hbase-git")
args = parser.parse_args()

# Setup logger
logging.basicConfig()
logger = logging.getLogger("submit-patch")
logger.setLevel(logging.INFO)


def log_fatal_and_exit(*arg):
    logger.fatal(*arg)
    sys.exit(1)


def assert_status_code(response, expected_status_code, description):
    if response.status_code != expected_status_code:
        log_fatal_and_exit(" Oops, something went wrong when %s. \nResponse: %s %s\nExiting..",
                           description, response.status_code, response.reason)


# Make repo instance to interact with git repo.
try:
    repo = git.Repo(os.getcwd())
    git = repo.git
except git.exc.InvalidGitRepositoryError as e:
    log_fatal_and_exit(" '%s' is not valid git repo directory.\nRun from base directory of "
                       "HBase's git repo.", e)

logger.info(" Active branch: %s", repo.active_branch.name)
# Do not proceed if there are uncommitted changes.
if repo.is_dirty():
    log_fatal_and_exit(" Git status is dirty. Commit locally first.")


# Returns base branch for creating diff.
def get_base_branch():
    # if --branch is set, use it as base branch for computing diff. Also check that it's a valid branch.
    if args.branch is not None:
        base_branch = args.branch
        # Check that given branch exists.
        for ref in repo.refs:
            if ref.name == base_branch:
                return base_branch
        log_fatal_and_exit(" Branch '%s' does not exist in refs.", base_branch)
    else:
        # if --branch is not set, use tracking branch as base branch for computing diff.
        # If there is no tracking branch, log error and quit.
        tracking_branch = repo.active_branch.tracking_branch()
        if tracking_branch is None:
            log_fatal_and_exit(" Active branch doesn't have a tracking_branch. Please specify base "
                               " branch for computing diff using --branch flag.")
        logger.info(" Using tracking branch as base branch")
        return tracking_branch.name


# Returns patch name having format (JIRA).(branch name).(patch number).patch. If no jira is
# specified, patch is name (branch name).patch.
def get_patch_name(branch):
    if args.jira_id is None:
        return branch + ".patch"

    patch_name_prefix = args.jira_id.upper() + "." + branch
    return get_patch_name_with_version(patch_name_prefix)


# Fetches list of attachments from the jira, deduces next version for the patch and returns final
# patch name.
def get_patch_name_with_version(patch_name_prefix):
    # JIRA's rest api is broken wrt to attachments. https://jira.atlassian.com/browse/JRA-27637.
    # Using crude way to get list of attachments.
    url = "https://issues.apache.org/jira/browse/" + args.jira_id
    logger.info("Getting list of attachments for jira %s from %s", args.jira_id, url)
    html = requests.get(url)
    if html.status_code == 404:
        log_fatal_and_exit(" Invalid jira id : %s", args.jira_id)
    if html.status_code != 200:
        log_fatal_and_exit(" Cannot fetch jira information. Status code %s", html.status_code)
    # Iterate over patch names starting from version 1 and return when name is not already used.
    content = str(html.content, 'utf-8')
    for i in range(1, 1000):
        name = patch_name_prefix + "." + ('{0:03d}'.format(i)) + ".patch"
        if name not in content:
            return name


# Validates that patch directory exists, if not, creates it.
def validate_patch_dir(patch_dir):
    # Create patch_dir if it doesn't exist.
    if not os.path.exists(patch_dir):
        logger.warn(" Patch directory doesn't exist. Creating it.")
        os.mkdir(patch_dir)
    else:
        # If patch_dir exists, make sure it's a directory.
        if not os.path.isdir(patch_dir):
            log_fatal_and_exit(" '%s' exists but is not a directory. Specify another directory.",
                               patch_dir)


# Make sure current branch is ahead of base_branch by exactly 1 commit. Quits if
# - base_branch has commits not in current branch
# - current branch is same as base branch
# - current branch is ahead of base_branch by more than 1 commits
def check_diff_between_branches(base_branch):
    only_in_base_branch = list(repo.iter_commits("HEAD.." + base_branch))
    only_in_active_branch = list(repo.iter_commits(base_branch + "..HEAD"))
    if len(only_in_base_branch) != 0:
        log_fatal_and_exit(" '%s' is ahead of current branch by %s commits. Rebase "
                           "and try again.", base_branch, len(only_in_base_branch))
    if len(only_in_active_branch) == 0:
        log_fatal_and_exit(" Current branch is same as '%s'. Exiting...", base_branch)
    if len(only_in_active_branch) > 1:
        log_fatal_and_exit(" Current branch is ahead of '%s' by %s commits. Squash into single "
                           "commit and try again.", base_branch, len(only_in_active_branch))


# If ~/.apache-creds is present, load credentials from it otherwise prompt user.
def get_credentials():
    creds = dict()
    creds_filepath = os.path.expanduser("~/.apache-creds")
    if os.path.exists(creds_filepath):
        try:
            logger.info(" Reading ~/.apache-creds for Jira and ReviewBoard credentials")
            content = subprocess.check_output("openssl enc -aes-256-cbc -d -in " + creds_filepath,
                                              shell=True)
        except subprocess.CalledProcessError as e:
            log_fatal_and_exit(" Couldn't decrypt ~/.apache-creds file. Exiting..")
        creds = json.loads(content)
    else:
        creds['jira_username'] = input("Jira username:")
        creds['jira_password'] = getpass.getpass("Jira password:")
        if not args.skip_review_board:
            creds['rb_username'] = input("Review Board username:")
            creds['rb_password'] = getpass.getpass("Review Board password:")
    return creds


def attach_patch_to_jira(issue_url, patch_filepath, patch_filename, creds):
    # Upload patch to jira using REST API.
    headers = {'X-Atlassian-Token': 'no-check'}
    files = {'file': (patch_filename, open(patch_filepath, 'rb'), 'text/plain')}
    jira_auth = requests.auth.HTTPBasicAuth(creds['jira_username'], creds['jira_password'])
    attachment_url = issue_url + "/attachments"
    r = requests.post(attachment_url, headers = headers, files = files, auth = jira_auth)
    assert_status_code(r, 200, "uploading patch to jira")


def get_jira_summary(issue_url):
    r = requests.get(issue_url + "?fields=summary")
    assert_status_code(r, 200, "fetching jira summary")
    return json.loads(r.content)["fields"]["summary"]


def get_review_board_id_if_present(issue_url, rb_link_title):
    r = requests.get(issue_url + "/remotelink")
    assert_status_code(r, 200, "fetching remote links")
    links = json.loads(r.content)
    for link in links:
        if link["object"]["title"] == rb_link_title:
            res = re.search("reviews.apache.org/r/([0-9]+)", link["object"]["url"])
            return res.group(1)
    return None


base_branch = get_base_branch()
# Remove remote repo name from branch name if present. This assumes that we don't use '/' in
# actual branch names.
base_branch_without_remote = base_branch.split('/')[-1]
logger.info(" Base branch: %s", base_branch)

check_diff_between_branches(base_branch)

patch_dir = os.path.abspath(os.path.expanduser(args.patch_dir))
logger.info(" Patch directory: %s", patch_dir)
validate_patch_dir(patch_dir)

patch_filename = get_patch_name(base_branch_without_remote)
logger.info(" Patch name: %s", patch_filename)
patch_filepath = os.path.join(patch_dir, patch_filename)

diff = git.format_patch(base_branch, stdout = True)
with open(patch_filepath, "w") as f:
    f.write(diff.encode('utf8'))

if args.jira_id is not None:
    creds = get_credentials()
    issue_url = "https://issues.apache.org/jira/rest/api/2/issue/" + args.jira_id

    attach_patch_to_jira(issue_url, patch_filepath, patch_filename, creds)

    if not args.skip_review_board:
        rb_auth = requests.auth.HTTPBasicAuth(creds['rb_username'], creds['rb_password'])

        rb_link_title = "Review Board (" + base_branch_without_remote + ")"
        rb_id = get_review_board_id_if_present(issue_url, rb_link_title)

        # If no review board link found, create new review request and add its link to jira.
        if rb_id is None:
            reviews_url = "https://reviews.apache.org/api/review-requests/"
            data = {"repository" : "hbase-git"}
            r = requests.post(reviews_url, data = data, auth = rb_auth)
            assert_status_code(r, 201, "creating new review request")
            review_request = json.loads(r.content)["review_request"]
            absolute_url = review_request["absolute_url"]
            logger.info(" Created new review request: %s", absolute_url)

            # Use jira summary as review's summary too.
            summary = get_jira_summary(issue_url)
            # Use commit message as description.
            description = repo.head.commit.message
            update_draft_data = {"bugs_closed" : [args.jira_id.upper()], "target_groups" : "hbase",
                                 "target_people" : args.reviewers, "summary" : summary,
                                 "description" : description }
            draft_url = review_request["links"]["draft"]["href"]
            r = requests.put(draft_url, data = update_draft_data, auth = rb_auth)
            assert_status_code(r, 200, "updating review draft")

            draft_request = json.loads(r.content)["draft"]
            diff_url = draft_request["links"]["draft_diffs"]["href"]
            files = {'path' : (patch_filename, open(patch_filepath, 'rb'))}
            r = requests.post(diff_url, files = files, auth = rb_auth)
            assert_status_code(r, 201, "uploading diff to review draft")

            r = requests.put(draft_url, data = {"public" : True}, auth = rb_auth)
            assert_status_code(r, 200, "publishing review request")

            # Add link to review board in the jira.
            remote_link = json.dumps({'object': {'url': absolute_url, 'title': rb_link_title}})
            jira_auth = requests.auth.HTTPBasicAuth(creds['jira_username'], creds['jira_password'])
            r = requests.post(issue_url + "/remotelink", data = remote_link, auth = jira_auth,
                              headers={'Content-Type':'application/json'})
        else:
            logger.info(" Updating existing review board: https://reviews.apache.org/r/%s", rb_id)
            draft_url = "https://reviews.apache.org/api/review-requests/" + rb_id + "/draft/"
            diff_url = draft_url + "diffs/"
            files = {'path' : (patch_filename, open(patch_filepath, 'rb'))}
            r = requests.post(diff_url, files = files, auth = rb_auth)
            assert_status_code(r, 201, "uploading diff to review draft")

            r = requests.put(draft_url, data = {"public" : True}, auth = rb_auth)
            assert_status_code(r, 200, "publishing review request")