Introduce issue migration script
This commit introduces the issue migration script for splitting x-pack into x-pack-elasticsearch, x-pack-kibana, and x-pack-logstash. Relates elastic/elasticsearch#4935 Original commit: elastic/x-pack-elasticsearch@33a00e5d06
This commit is contained in:
Jason Tedor
GitHub
parent
e51b850d75
commit
be3b5f49d0
|
|
@ -0,0 +1,194 @@
|
||||||
|
import argparse
|
||||||
|
import calendar
|
||||||
|
import json
|
||||||
|
import re
|
||||||
|
import requests
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
import tempfile
|
||||||
|
import time
|
||||||
|
|
||||||
|
def throttle(response):
|
||||||
|
if 'X-RateLimit-Remaining' in response.headers:
|
||||||
|
rate_limit_remaining = int(response.headers['X-RateLimit-Remaining'])
|
||||||
|
if rate_limit_remaining < 16:
|
||||||
|
rate_limit_reset = int(response.headers['X-RateLimit-Reset'])
|
||||||
|
delay = rate_limit_reset - calendar.timegm(time.gmtime())
|
||||||
|
print('rate limit remaining: {}, rate limit reset: {}, throttling for {} seconds'.format(rate_limit_remaining, rate_limit_reset, delay))
|
||||||
|
if delay >= 0:
|
||||||
|
time.sleep(delay)
|
||||||
|
|
||||||
|
return response
|
||||||
|
|
||||||
|
def authorization_token(token):
|
||||||
|
return {'Authorization':'token {}'.format(token)}
|
||||||
|
|
||||||
|
def next_url(response):
|
||||||
|
"""Return the next URL following the Link header, otherwise None."""
|
||||||
|
nu = None
|
||||||
|
if 'Link' in response.headers:
|
||||||
|
links = response.headers['Link'].split(',')
|
||||||
|
for link in links:
|
||||||
|
if 'rel="next"' in link:
|
||||||
|
nu = link.split(';')[0][1:-1]
|
||||||
|
break
|
||||||
|
|
||||||
|
return nu
|
||||||
|
|
||||||
|
def get(url, token):
|
||||||
|
"""Return the response for the specified URL."""
|
||||||
|
headers = authorization_token(token)
|
||||||
|
return throttle(requests.get(url, headers=headers))
|
||||||
|
|
||||||
|
def get_all(url, token):
|
||||||
|
"""Returns all pages starting at the specified URL."""
|
||||||
|
items = []
|
||||||
|
while url is not None:
|
||||||
|
response = get(url, token)
|
||||||
|
json = response.json()
|
||||||
|
if json:
|
||||||
|
items.extend(response.json())
|
||||||
|
url = next_url(response)
|
||||||
|
else:
|
||||||
|
url = None
|
||||||
|
|
||||||
|
return items
|
||||||
|
|
||||||
|
def issue_comments(source_owner, source_repo, issue, token):
|
||||||
|
"""Return all issue comments."""
|
||||||
|
url = 'https://api.github.com/repos/{}/{}/issues/{}/comments'.format(source_owner, source_repo, issue)
|
||||||
|
return get_all(url, token)
|
||||||
|
|
||||||
|
assignees_cache = {}
|
||||||
|
|
||||||
|
def repo_assignees(assignee, owner, repo, token):
|
||||||
|
"""Returns True if the assignee is valid for the specified owner/repo, otherwise False."""
|
||||||
|
if assignee in assignees_cache:
|
||||||
|
return assignees_cache[assignee]
|
||||||
|
url = 'https://api.github.com/repos/{}/{}/assignees/{}'.format(owner, repo, assignee)
|
||||||
|
response = get(url, token)
|
||||||
|
assignees_cache[assignee] = response.status_code == 204
|
||||||
|
return assignees_cache[assignee]
|
||||||
|
|
||||||
|
def rewrite_issue_links(text, source_owner, source_repo):
|
||||||
|
return re.sub(r"(\s+)(#\d+)", "\\1{}/{}\\2".format(source_owner, source_repo), text)
|
||||||
|
|
||||||
|
def rewrite_commits(text, source_owner, source_repo, temp):
|
||||||
|
commits = []
|
||||||
|
for match in re.finditer(r"(?<!@)[a-f0-9]{7,40}", text):
|
||||||
|
if subprocess.call(['git', '-C', temp, 'cat-file', 'commit', match.group(0)], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) == 0:
|
||||||
|
commits.append(match.group(0))
|
||||||
|
|
||||||
|
for commit in commits:
|
||||||
|
text = re.sub(r"(?<!@)({})".format(commit), "{}/{}@{}".format(source_owner, source_repo, commit), text)
|
||||||
|
|
||||||
|
return text
|
||||||
|
|
||||||
|
def copy_issue(issue, label_names, source_owner, source_repo, destination_owner, destination_repo, token, temp):
|
||||||
|
"""Posts an issue to the specified owner/repo with the specified labels."""
|
||||||
|
url = 'https://api.github.com/repos/{}/{}/issues'.format(destination_owner, destination_repo)
|
||||||
|
headers = authorization_token(token)
|
||||||
|
issue_number = issue['number']
|
||||||
|
title = issue['title']
|
||||||
|
issue_body = rewrite_commits(rewrite_issue_links(issue['body'], source_owner, source_repo), source_owner, source_repo, temp)
|
||||||
|
body = '*Original comment by @{}:*'.format(issue['user']['login']) + '\n\n' + issue_body + '\n\n' + 'Supercedes {}/{}#{}'.format(source_owner, source_repo, issue_number)
|
||||||
|
labels = [label for label in label_names]
|
||||||
|
assignees = [assignee['login'] for assignee in issue['assignees'] if repo_assignees(assignee['login'], destination_owner, destination_repo, token)]
|
||||||
|
payload = { 'title':title, 'body':body, 'labels':labels, 'assignees':assignees }
|
||||||
|
return throttle(requests.post(url, headers=headers, data=json.dumps(payload)))
|
||||||
|
|
||||||
|
def close_issue(issue_number, owner, repo, token):
|
||||||
|
url = 'https://api.github.com/repos/{}/{}/issues/{}'.format(owner, repo, issue_number)
|
||||||
|
headers = authorization_token(token)
|
||||||
|
payload = { 'state':'closed' }
|
||||||
|
return throttle(requests.post(url, headers=headers, data=json.dumps(payload)))
|
||||||
|
|
||||||
|
def copy_comment(comment, source_owner, source_repo, destination_owner, destination_repo, issue_number, token, temp):
|
||||||
|
"""Copies a comment to the specified issue in the owner/repo."""
|
||||||
|
comment_body = rewrite_commits(rewrite_issue_links(comment['body'], source_owner, source_repo), source_owner, source_repo, temp)
|
||||||
|
body = '*Original comment by @{}:*'.format(comment['user']['login']) + '\n\n' + comment_body
|
||||||
|
return post_comment(body, destination_owner, destination_repo, issue_number, token)
|
||||||
|
|
||||||
|
def post_comment(body, destination_owner, destination_repo, issue_number, token):
|
||||||
|
"""Posts a comment to the specified issue in the owner/repo."""
|
||||||
|
url = 'https://api.github.com/repos/{}/{}/issues/{}/comments'.format(destination_owner, destination_repo, issue_number)
|
||||||
|
headers = authorization_token(token)
|
||||||
|
payload = { 'body':body }
|
||||||
|
return throttle(requests.post(url, headers=headers, data=json.dumps(payload)))
|
||||||
|
|
||||||
|
def post_label(name, color, destination_owner, destination_repo, token):
|
||||||
|
url = 'https://api.github.com/repos/{}/{}/labels/{}'.format(destination_owner, destination_repo, name)
|
||||||
|
headers = authorization_token(token)
|
||||||
|
payload = { 'name':name, 'color':color }
|
||||||
|
response = throttle(requests.post(url, headers=headers, data=json.dumps(payload)))
|
||||||
|
if response.status_code != 200:
|
||||||
|
raise('updating label {} failed'.format(name))
|
||||||
|
|
||||||
|
def merge_labels(existing_labels, new_labels):
|
||||||
|
existing_labels.update({label['name']:label['color'] for label in new_labels})
|
||||||
|
|
||||||
|
def update_labels(labels, destination_owner, destination_repo, token):
|
||||||
|
for name, color in labels.items():
|
||||||
|
post_label(name, color, destination_owner, destination_repo, token)
|
||||||
|
|
||||||
|
def main(source_owner, source_repo, destination_owner, kibana_destination_repo, logstash_destination_repo, elasticsearch_destination_repo, token):
|
||||||
|
temp = tempfile.mkdtemp()
|
||||||
|
subprocess.call(['git', 'clone', 'https://{}@github.com/{}/{}'.format(token, source_owner, source_repo), temp], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||||
|
try:
|
||||||
|
kibana_labels = {}
|
||||||
|
logstash_labels = {}
|
||||||
|
elasticsearch_labels = {}
|
||||||
|
url = 'https://api.github.com/repos/{}/{}/issues?direction=asc'.format(source_owner, source_repo)
|
||||||
|
while url is not None:
|
||||||
|
issues = get(url, token)
|
||||||
|
for issue in [i for i in issues.json() if 'pull_request' not in i]:
|
||||||
|
# issue_number represents the issue number in the source repo
|
||||||
|
issue_number = issue['number']
|
||||||
|
print('processing issue: {}'.format(issue_number))
|
||||||
|
labels = issue['labels']
|
||||||
|
label_names = {label['name'] for label in labels}
|
||||||
|
if ':UI' in label_names or ':reporting' in label_names:
|
||||||
|
merge_labels(kibana_labels, labels)
|
||||||
|
destination_repo = kibana_destination_repo
|
||||||
|
elif ':logstash' in label_names:
|
||||||
|
merge_labels(logstash_labels, labels)
|
||||||
|
destination_repo = logstash_destination_repo
|
||||||
|
else:
|
||||||
|
merge_labels(elasticsearch_labels, labels)
|
||||||
|
destination_repo = elasticsearch_destination_repo
|
||||||
|
new_issue = copy_issue(issue, label_names, source_owner, source_repo, destination_owner, destination_repo, token, temp)
|
||||||
|
# new_issue_number represents the issue number in the destination repo√
|
||||||
|
new_issue_number = new_issue.json()['number']
|
||||||
|
print('issue {} copied to new issue {} in {}/{}'.format(issue_number, new_issue_number, destination_owner, destination_repo))
|
||||||
|
# post each comment from the source issue to the destination issue
|
||||||
|
comments = issue_comments(source_owner, source_repo, issue_number, token)
|
||||||
|
for comment in comments:
|
||||||
|
copy_comment(comment, source_owner, source_repo, destination_owner, destination_repo, new_issue_number, token, temp)
|
||||||
|
|
||||||
|
# comment on the original issue referring to the new issue
|
||||||
|
post_comment('Superceded by {}/{}#{}'.format(destination_owner, destination_repo, new_issue_number), source_owner, source_repo, issue_number, token)
|
||||||
|
close_issue(issue_number, source_owner, source_repo, token)
|
||||||
|
print('issue {} closed'.format(issue_number))
|
||||||
|
|
||||||
|
# proceed to the next page of issues
|
||||||
|
url = next_url(issues)
|
||||||
|
|
||||||
|
# update labels
|
||||||
|
update_labels(kibana_labels, destination_owner, kibana_destination_repo, token)
|
||||||
|
update_labels(logstash_labels, destination_owner, logstash_destination_repo, token)
|
||||||
|
update_labels(elasticsearch_labels, destination_owner, elasticsearch_destination_repo, token)
|
||||||
|
|
||||||
|
finally:
|
||||||
|
shutil.rmtree(temp)
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
parser = argparse.ArgumentParser(description='Migrate issues')
|
||||||
|
parser.add_argument('--source_owner', required=True)
|
||||||
|
parser.add_argument('--source_repo', required=True)
|
||||||
|
parser.add_argument('--destination_owner', required=True)
|
||||||
|
parser.add_argument('--kibana_destination_repo', required=True)
|
||||||
|
parser.add_argument('--logstash_destination_repo', required=True)
|
||||||
|
parser.add_argument('--elasticsearch_destination_repo', required=True)
|
||||||
|
parser.add_argument('--token', required=True)
|
||||||
|
args = parser.parse_args()
|
||||||
|
main(args.source_owner, args.source_repo, args.destination_owner, args.kibana_destination_repo, args.logstash_destination_repo, args.elasticsearch_destination_repo, args.token)
|
||||||
Loading…
Reference in New Issue