import argparse import calendar import json import re import requests import shutil import subprocess import tempfile import time def throttle(response): if 'X-RateLimit-Remaining' in response.headers: rate_limit_remaining = int(response.headers['X-RateLimit-Remaining']) if rate_limit_remaining < 16: rate_limit_reset = int(response.headers['X-RateLimit-Reset']) delay = rate_limit_reset - calendar.timegm(time.gmtime()) print('rate limit remaining: {}, rate limit reset: {}, throttling for {} seconds'.format(rate_limit_remaining, rate_limit_reset, delay)) if delay >= 0: time.sleep(delay) return response def authorization_token(token): return {'Authorization':'token {}'.format(token)} def next_url(response): """Return the next URL following the Link header, otherwise None.""" nu = None if 'Link' in response.headers: links = response.headers['Link'].split(',') for link in links: if 'rel="next"' in link: nu = link.split(';')[0][1:-1] break return nu def get(url, token): """Return the response for the specified URL.""" headers = authorization_token(token) return throttle(requests.get(url, headers=headers)) def get_all(url, token): """Returns all pages starting at the specified URL.""" items = [] while url is not None: response = get(url, token) json = response.json() if json: items.extend(response.json()) url = next_url(response) else: url = None return items def issue_comments(source_owner, source_repo, issue, token): """Return all issue comments.""" url = 'https://api.github.com/repos/{}/{}/issues/{}/comments'.format(source_owner, source_repo, issue) return get_all(url, token) assignees_cache = {} def repo_assignees(assignee, owner, repo, token): """Returns True if the assignee is valid for the specified owner/repo, otherwise False.""" if assignee in assignees_cache: return assignees_cache[assignee] url = 'https://api.github.com/repos/{}/{}/assignees/{}'.format(owner, repo, assignee) response = get(url, token) assignees_cache[assignee] = response.status_code == 204 return assignees_cache[assignee] def rewrite_issue_links(text, source_owner, source_repo): return re.sub(r"(\s+)(#\d+)", "\\1{}/{}\\2".format(source_owner, source_repo), text) def rewrite_commits(text, source_owner, source_repo, temp): commits = [] for match in re.finditer(r"(?<!@)[a-f0-9]{7,40}", text): if subprocess.call(['git', '-C', temp, 'cat-file', 'commit', match.group(0)], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) == 0: commits.append(match.group(0)) for commit in commits: text = re.sub(r"(?<!@)({})".format(commit), "{}/{}@{}".format(source_owner, source_repo, commit), text) return text def copy_issue(issue, label_names, source_owner, source_repo, destination_owner, destination_repo, token, temp): """Posts an issue to the specified owner/repo with the specified labels.""" url = 'https://api.github.com/repos/{}/{}/issues'.format(destination_owner, destination_repo) headers = authorization_token(token) issue_number = issue['number'] title = issue['title'] issue_body = rewrite_commits(rewrite_issue_links(issue['body'], source_owner, source_repo), source_owner, source_repo, temp) body = '*Original comment by @{}:*'.format(issue['user']['login']) + '\n\n' + issue_body + '\n\n' + 'Supercedes {}/{}#{}'.format(source_owner, source_repo, issue_number) labels = [label for label in label_names] assignees = [assignee['login'] for assignee in issue['assignees'] if repo_assignees(assignee['login'], destination_owner, destination_repo, token)] payload = { 'title':title, 'body':body, 'labels':labels, 'assignees':assignees } return throttle(requests.post(url, headers=headers, data=json.dumps(payload))) def close_issue(issue_number, owner, repo, token): url = 'https://api.github.com/repos/{}/{}/issues/{}'.format(owner, repo, issue_number) headers = authorization_token(token) payload = { 'state':'closed' } return throttle(requests.post(url, headers=headers, data=json.dumps(payload))) def copy_comment(comment, source_owner, source_repo, destination_owner, destination_repo, issue_number, token, temp): """Copies a comment to the specified issue in the owner/repo.""" comment_body = rewrite_commits(rewrite_issue_links(comment['body'], source_owner, source_repo), source_owner, source_repo, temp) body = '*Original comment by @{}:*'.format(comment['user']['login']) + '\n\n' + comment_body return post_comment(body, destination_owner, destination_repo, issue_number, token) def post_comment(body, destination_owner, destination_repo, issue_number, token): """Posts a comment to the specified issue in the owner/repo.""" url = 'https://api.github.com/repos/{}/{}/issues/{}/comments'.format(destination_owner, destination_repo, issue_number) headers = authorization_token(token) payload = { 'body':body } return throttle(requests.post(url, headers=headers, data=json.dumps(payload))) def post_label(name, color, destination_owner, destination_repo, token): url = 'https://api.github.com/repos/{}/{}/labels/{}'.format(destination_owner, destination_repo, name) headers = authorization_token(token) payload = { 'name':name, 'color':color } response = throttle(requests.post(url, headers=headers, data=json.dumps(payload))) if response.status_code != 200: raise('updating label {} failed'.format(name)) def merge_labels(existing_labels, new_labels): existing_labels.update({label['name']:label['color'] for label in new_labels}) def update_labels(labels, destination_owner, destination_repo, token): for name, color in labels.items(): post_label(name, color, destination_owner, destination_repo, token) def main(source_owner, source_repo, destination_owner, kibana_destination_repo, logstash_destination_repo, elasticsearch_destination_repo, token): temp = tempfile.mkdtemp() subprocess.call(['git', 'clone', 'https://{}@github.com/{}/{}'.format(token, source_owner, source_repo), temp], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) try: kibana_labels = {} logstash_labels = {} elasticsearch_labels = {} url = 'https://api.github.com/repos/{}/{}/issues?direction=asc'.format(source_owner, source_repo) while url is not None: issues = get(url, token) for issue in [i for i in issues.json() if 'pull_request' not in i]: # issue_number represents the issue number in the source repo issue_number = issue['number'] print('processing issue: {}'.format(issue_number)) labels = issue['labels'] label_names = {label['name'] for label in labels} if ':UI' in label_names or ':reporting' in label_names: merge_labels(kibana_labels, labels) destination_repo = kibana_destination_repo elif ':logstash' in label_names: merge_labels(logstash_labels, labels) destination_repo = logstash_destination_repo else: merge_labels(elasticsearch_labels, labels) destination_repo = elasticsearch_destination_repo new_issue = copy_issue(issue, label_names, source_owner, source_repo, destination_owner, destination_repo, token, temp) # new_issue_number represents the issue number in the destination repo√ new_issue_number = new_issue.json()['number'] print('issue {} copied to new issue {} in {}/{}'.format(issue_number, new_issue_number, destination_owner, destination_repo)) # post each comment from the source issue to the destination issue comments = issue_comments(source_owner, source_repo, issue_number, token) for comment in comments: copy_comment(comment, source_owner, source_repo, destination_owner, destination_repo, new_issue_number, token, temp) # comment on the original issue referring to the new issue post_comment('Superceded by {}/{}#{}'.format(destination_owner, destination_repo, new_issue_number), source_owner, source_repo, issue_number, token) close_issue(issue_number, source_owner, source_repo, token) print('issue {} closed'.format(issue_number)) # proceed to the next page of issues url = next_url(issues) # update labels update_labels(kibana_labels, destination_owner, kibana_destination_repo, token) update_labels(logstash_labels, destination_owner, logstash_destination_repo, token) update_labels(elasticsearch_labels, destination_owner, elasticsearch_destination_repo, token) finally: shutil.rmtree(temp) if __name__ == "__main__": parser = argparse.ArgumentParser(description='Migrate issues') parser.add_argument('--source_owner', required=True) parser.add_argument('--source_repo', required=True) parser.add_argument('--destination_owner', required=True) parser.add_argument('--kibana_destination_repo', required=True) parser.add_argument('--logstash_destination_repo', required=True) parser.add_argument('--elasticsearch_destination_repo', required=True) parser.add_argument('--token', required=True) args = parser.parse_args() main(args.source_owner, args.source_repo, args.destination_owner, args.kibana_destination_repo, args.logstash_destination_repo, args.elasticsearch_destination_repo, args.token)