From f2840c5ccfd87942ca17489f12ac6e3afc0bf10b Mon Sep 17 00:00:00 2001 From: Simon Willnauer Date: Mon, 4 Nov 2013 21:11:33 +0100 Subject: [PATCH] Added doc file processing to update coming/added references We use 'coming[x.y.z]' in our ref docs which needs to be updated when doing a release to 'added[x.y.z]' This commit adds support for replacing the references where applicable during preparing the release. --- dev-tools/build_release.py | 86 +++++++++++++++++++++++++++++++------- 1 file changed, 70 insertions(+), 16 deletions(-) diff --git a/dev-tools/build_release.py b/dev-tools/build_release.py index bd64db4f1c9..c7b921be5ac 100644 --- a/dev-tools/build_release.py +++ b/dev-tools/build_release.py @@ -24,6 +24,7 @@ import sys import argparse import hmac import urllib +import fnmatch from http.client import HTTPConnection """ @@ -95,42 +96,80 @@ def verify_java_version(version): if s.find(' version "%s.' % version) == -1: raise RuntimeError('got wrong version for java %s:\n%s' % (version, s)) +# Verifies the java version. We guarantee that we run with Java 1.6 +# If 1.6 is not available fail the build! def verify_mvn_java_version(version, mvn): s = os.popen('%s; %s --version 2>&1' % (java_exe(), mvn)).read() if s.find('Java version: %s' % version) == -1: raise RuntimeError('got wrong java version for %s %s:\n%s' % (mvn, version, s)) +# Returns the hash of the current git HEAD revision def get_head_hash(): return os.popen('git rev-parse --verify HEAD 2>&1').read().strip() +# Returns the name of the current branch def get_current_branch(): return os.popen('git rev-parse --abbrev-ref HEAD 2>&1').read().strip() verify_java_version('1.6') # we require to build with 1.6 verify_mvn_java_version('1.6', MVN) -def release_branch(release): - return 'release_branch_%s' % (release) +# Utility that returns the name of the release branch for a given version +def release_branch(version): + return 'release_branch_%s' % version +# Creates a new release branch from the given source branch +# and rebases the source branch from the remote before creating +# the release branch. Note: This fails if the source branch +# doesn't exist on the provided remote. def create_release_branch(remote, src_branch, release): - run('git checkout %s' % (src_branch)) + run('git checkout %s' % src_branch) run('git pull --rebase %s %s' % (remote, src_branch)) run('git checkout -b %s' % (release_branch(release))) +# Reads the given file and applies the +# callback to it. If the callback changed +# a line the given file is replaced with +# the modified input. def process_file(file_path, line_callback): fh, abs_path = tempfile.mkstemp() + modified = False with open(abs_path,'w') as new_file: with open(file_path) as old_file: for line in old_file: - new_file.write(line_callback(line)) + new_line = line_callback(line) + modified = modified or (new_line != line) + new_file.write(new_line) os.close(fh) - #Remove original file - os.remove(file_path) - #Move new file - shutil.move(abs_path, file_path) - + if modified: + #Remove original file + os.remove(file_path) + #Move new file + shutil.move(abs_path, file_path) + return True + else: + # nothing to do - just remove the tmp file + os.remove(abs_path) + return False +# Walks the given directory path (defaults to 'docs') +# and replaces all 'coming[$version]' tags with +# 'added[$version]'. This method only accesses asciidoc files. +def update_reference_docs(release_version, path='docs'): + pattern = 'coming[%s' % (release_version) + replacement = 'added[%s' % (release_version) + pending_files = [] + def callback(line): + return line.replace(pattern, replacement) + for root, _, file_names in os.walk(path): + for file_name in fnmatch.filter(file_names, '*.asciidoc'): + full_path = os.path.join(root, file_name) + if process_file(full_path, callback): + pending_files.append(os.path.join(root, file_name)) + return pending_files + +# Moves the pom.xml file from a snapshot to a release def remove_maven_snapshot(pom, release): pattern = '%s-SNAPSHOT' % (release) replacement = '%s' % (release) @@ -138,7 +177,7 @@ def remove_maven_snapshot(pom, release): return line.replace(pattern, replacement) process_file(pom, callback) - +# Moves the Version.java file from a snapshot to a release def remove_version_snapshot(version_file, release): # 1.0.0.Beta1 -> 1_0_0_Beat1 release = release.replace('.', '_') @@ -148,10 +187,12 @@ def remove_version_snapshot(version_file, release): return line.replace(pattern, replacement) process_file(version_file, callback) +# Stages the given files for the next git commit def add_pending_files(*files): for file in files: run('git add %s' % (file)) +# Executes a git commit with 'release [version]' as the commit message def commit_release(release): run('git commit -m "release [%s]"' % release) @@ -161,6 +202,7 @@ def tag_release(release): def run_mvn(*cmd): for c in cmd: run('%s; %s %s' % (java_exe(), MVN, c)) + def build_release(run_tests=False, dry_run=True, cpus=1): target = 'deploy' if dry_run: @@ -176,28 +218,38 @@ def wait_for_node_startup(host='127.0.0.1', port=9200,timeout=15): conn = HTTPConnection(host, port, timeout); for _ in range(timeout): try: + log('Waiting until node becomes available for 1 second') time.sleep(1) + log('Check if node is available') conn.request('GET', '') res = conn.getresponse() if res.status == 200: return True - except: - pass #that is ok it might not be there yet + except Exception as e: + log("Failed while waiting for node - Exception: [%s]" % (e.message)) + #that is ok it might not be there yet return False +# Checks the pom.xml for the release version. +# This method fails if the pom file has no SNAPSHOT version set ie. +# if the version is already on a release version we fail. +# Returns the next version string ie. 0.90.7 def find_release_version(src_branch): - run('git checkout %s' % (src_branch)) + run('git checkout %s' % src_branch) with open('pom.xml') as file: for line in file: match = re.search(r'(.+)-SNAPSHOT', line) if match: return match.group(1) - raise RuntimeError('Could not find release version in branch %s' % (src_branch)) + raise RuntimeError('Could not find release version in branch %s' % src_branch) def get_artifacts(release, path='target/releases/'): return [os.path.join(path, 'elasticsearch-%s.%s' % (release, t)) for t in ['deb', 'tar.gz', 'zip']] +# Generates sha1 checsums for all files +# and returns the checksum files as well +# as the given files in a list def generate_checksums(files): res = [] for release_file in files: @@ -351,12 +403,14 @@ if __name__ == '__main__': print(' Created release branch [%s]' % (release_branch(release_version))) success = False try: + pending_files = [POM_FILE, VERSION_FILE] remove_maven_snapshot(POM_FILE, release_version) remove_version_snapshot(VERSION_FILE, release_version) + pending_files = pending_files + update_reference_docs(release_version) print(' Done removing snapshot version') - add_pending_files(VERSION_FILE, POM_FILE) + add_pending_files(*pending_files) # expects var args use * to expand commit_release(release_version) - print(' Committed release version [%s]' % (release_version)) + print(' Committed release version [%s]' % release_version) print(''.join(['-' for _ in range(80)])) print('Building Release candidate') input('Press Enter to continue...')