honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Introduce issue migration script 

This commit introduces the issue migration script for splitting x-pack
into x-pack-elasticsearch, x-pack-kibana, and x-pack-logstash.

Relates elastic/elasticsearch#4935

Original commit: elastic/x-pack-elasticsearch@33a00e5d06
This commit is contained in:
Jason Tedor 2017-02-09 17:09:30 -05:00 committed by GitHub
parent e51b850d75
commit be3b5f49d0
1 changed files with 194 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

194
migrate-issues.py Normal file
View File

@ -0,0 +1,194 @@
import argparse
import calendar
import json
import re
import requests
import shutil
import subprocess
import tempfile
import time
def throttle(response):
if 'X-RateLimit-Remaining' in response.headers:
rate_limit_remaining = int(response.headers['X-RateLimit-Remaining'])
if rate_limit_remaining < 16:
rate_limit_reset = int(response.headers['X-RateLimit-Reset'])
delay = rate_limit_reset - calendar.timegm(time.gmtime())
print('rate limit remaining: {}, rate limit reset: {}, throttling for {} seconds'.format(rate_limit_remaining, rate_limit_reset, delay))
if delay >= 0:
time.sleep(delay)
return response
def authorization_token(token):
return {'Authorization':'token {}'.format(token)}
def next_url(response):
"""Return the next URL following the Link header, otherwise None."""
nu = None
if 'Link' in response.headers:
links = response.headers['Link'].split(',')
for link in links:
if 'rel="next"' in link:
nu = link.split(';')[0][1:-1]
break
return nu
def get(url, token):
"""Return the response for the specified URL."""
headers = authorization_token(token)
return throttle(requests.get(url, headers=headers))
def get_all(url, token):
"""Returns all pages starting at the specified URL."""
items = []
while url is not None:
response = get(url, token)
json = response.json()
if json:
items.extend(response.json())
url = next_url(response)
else:
url = None
return items
def issue_comments(source_owner, source_repo, issue, token):
"""Return all issue comments."""
url = 'https://api.github.com/repos/{}/{}/issues/{}/comments'.format(source_owner, source_repo, issue)
return get_all(url, token)
assignees_cache = {}
def repo_assignees(assignee, owner, repo, token):
"""Returns True if the assignee is valid for the specified owner/repo, otherwise False."""
if assignee in assignees_cache:
return assignees_cache[assignee]
url = 'https://api.github.com/repos/{}/{}/assignees/{}'.format(owner, repo, assignee)
response = get(url, token)
assignees_cache[assignee] = response.status_code == 204
return assignees_cache[assignee]
def rewrite_issue_links(text, source_owner, source_repo):
return re.sub(r"(\s+)(#\d+)", "\\1{}/{}\\2".format(source_owner, source_repo), text)
def rewrite_commits(text, source_owner, source_repo, temp):
commits = []
for match in re.finditer(r"(?<!@)[a-f0-9]{7,40}", text):
if subprocess.call(['git', '-C', temp, 'cat-file', 'commit', match.group(0)], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) == 0:
commits.append(match.group(0))
for commit in commits:
text = re.sub(r"(?<!@)({})".format(commit), "{}/{}@{}".format(source_owner, source_repo, commit), text)
return text
def copy_issue(issue, label_names, source_owner, source_repo, destination_owner, destination_repo, token, temp):
"""Posts an issue to the specified owner/repo with the specified labels."""
url = 'https://api.github.com/repos/{}/{}/issues'.format(destination_owner, destination_repo)
headers = authorization_token(token)
issue_number = issue['number']
title = issue['title']
issue_body = rewrite_commits(rewrite_issue_links(issue['body'], source_owner, source_repo), source_owner, source_repo, temp)
body = '*Original comment by @{}:*'.format(issue['user']['login']) + '\n\n' + issue_body + '\n\n' + 'Supercedes {}/{}#{}'.format(source_owner, source_repo, issue_number)
labels = [label for label in label_names]
assignees = [assignee['login'] for assignee in issue['assignees'] if repo_assignees(assignee['login'], destination_owner, destination_repo, token)]
payload = { 'title':title, 'body':body, 'labels':labels, 'assignees':assignees }
return throttle(requests.post(url, headers=headers, data=json.dumps(payload)))
def close_issue(issue_number, owner, repo, token):
url = 'https://api.github.com/repos/{}/{}/issues/{}'.format(owner, repo, issue_number)
headers = authorization_token(token)
payload = { 'state':'closed' }
return throttle(requests.post(url, headers=headers, data=json.dumps(payload)))
def copy_comment(comment, source_owner, source_repo, destination_owner, destination_repo, issue_number, token, temp):
"""Copies a comment to the specified issue in the owner/repo."""
comment_body = rewrite_commits(rewrite_issue_links(comment['body'], source_owner, source_repo), source_owner, source_repo, temp)
body = '*Original comment by @{}:*'.format(comment['user']['login']) + '\n\n' + comment_body
return post_comment(body, destination_owner, destination_repo, issue_number, token)
def post_comment(body, destination_owner, destination_repo, issue_number, token):
"""Posts a comment to the specified issue in the owner/repo."""
url = 'https://api.github.com/repos/{}/{}/issues/{}/comments'.format(destination_owner, destination_repo, issue_number)
headers = authorization_token(token)
payload = { 'body':body }
return throttle(requests.post(url, headers=headers, data=json.dumps(payload)))
def post_label(name, color, destination_owner, destination_repo, token):
url = 'https://api.github.com/repos/{}/{}/labels/{}'.format(destination_owner, destination_repo, name)
headers = authorization_token(token)
payload = { 'name':name, 'color':color }
response = throttle(requests.post(url, headers=headers, data=json.dumps(payload)))
if response.status_code != 200:
raise('updating label {} failed'.format(name))
def merge_labels(existing_labels, new_labels):
existing_labels.update({label['name']:label['color'] for label in new_labels})
def update_labels(labels, destination_owner, destination_repo, token):
for name, color in labels.items():
post_label(name, color, destination_owner, destination_repo, token)
def main(source_owner, source_repo, destination_owner, kibana_destination_repo, logstash_destination_repo, elasticsearch_destination_repo, token):
temp = tempfile.mkdtemp()
subprocess.call(['git', 'clone', 'https://{}@github.com/{}/{}'.format(token, source_owner, source_repo), temp], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
try:
kibana_labels = {}
logstash_labels = {}
elasticsearch_labels = {}
url = 'https://api.github.com/repos/{}/{}/issues?direction=asc'.format(source_owner, source_repo)
while url is not None:
issues = get(url, token)
for issue in [i for i in issues.json() if 'pull_request' not in i]:
# issue_number represents the issue number in the source repo
issue_number = issue['number']
print('processing issue: {}'.format(issue_number))
labels = issue['labels']
label_names = {label['name'] for label in labels}
if ':UI' in label_names or ':reporting' in label_names:
merge_labels(kibana_labels, labels)
destination_repo = kibana_destination_repo
elif ':logstash' in label_names:
merge_labels(logstash_labels, labels)
destination_repo = logstash_destination_repo
else:
merge_labels(elasticsearch_labels, labels)
destination_repo = elasticsearch_destination_repo
new_issue = copy_issue(issue, label_names, source_owner, source_repo, destination_owner, destination_repo, token, temp)
# new_issue_number represents the issue number in the destination repo√
new_issue_number = new_issue.json()['number']
print('issue {} copied to new issue {} in {}/{}'.format(issue_number, new_issue_number, destination_owner, destination_repo))
# post each comment from the source issue to the destination issue
comments = issue_comments(source_owner, source_repo, issue_number, token)
for comment in comments:
copy_comment(comment, source_owner, source_repo, destination_owner, destination_repo, new_issue_number, token, temp)
# comment on the original issue referring to the new issue
post_comment('Superceded by {}/{}#{}'.format(destination_owner, destination_repo, new_issue_number), source_owner, source_repo, issue_number, token)
close_issue(issue_number, source_owner, source_repo, token)
print('issue {} closed'.format(issue_number))
# proceed to the next page of issues
url = next_url(issues)
# update labels
update_labels(kibana_labels, destination_owner, kibana_destination_repo, token)
update_labels(logstash_labels, destination_owner, logstash_destination_repo, token)
update_labels(elasticsearch_labels, destination_owner, elasticsearch_destination_repo, token)
finally:
shutil.rmtree(temp)
if __name__ == "__main__":
parser = argparse.ArgumentParser(description='Migrate issues')
parser.add_argument('--source_owner', required=True)
parser.add_argument('--source_repo', required=True)
parser.add_argument('--destination_owner', required=True)
parser.add_argument('--kibana_destination_repo', required=True)
parser.add_argument('--logstash_destination_repo', required=True)
parser.add_argument('--elasticsearch_destination_repo', required=True)
parser.add_argument('--token', required=True)
args = parser.parse_args()
main(args.source_owner, args.source_repo, args.destination_owner, args.kibana_destination_repo, args.logstash_destination_repo, args.elasticsearch_destination_repo, args.token)