Added doc file processing to update coming/added references

We use 'coming[x.y.z]' in our ref docs which needs to be updated
when doing a release to 'added[x.y.z]' This commit adds support
for replacing the references where applicable during preparing the release.

dev-tools/build_release.py

@@ -24,6 +24,7 @@ import sys
 import argparse
 import hmac
 import urllib
+import fnmatch
 from http.client import HTTPConnection
 
 """ 
@@ -95,42 +96,80 @@ def verify_java_version(version):
   if s.find(' version "%s.' % version) == -1:
     raise RuntimeError('got wrong version for java %s:\n%s' % (version, s))
 
+# Verifies the java version. We guarantee that we run with Java 1.6
+# If 1.6 is not available fail the build!
 def verify_mvn_java_version(version, mvn):
   s = os.popen('%s; %s --version 2>&1' % (java_exe(), mvn)).read()
   if s.find('Java version: %s' % version) == -1:
     raise RuntimeError('got wrong java version for %s %s:\n%s' % (mvn, version, s))
 
+# Returns the hash of the current git HEAD revision
 def get_head_hash():
   return os.popen('git rev-parse --verify HEAD 2>&1').read().strip()
 
+# Returns the name of the current branch
 def get_current_branch():
   return os.popen('git rev-parse --abbrev-ref HEAD  2>&1').read().strip()
 
 verify_java_version('1.6') # we require to build with 1.6
 verify_mvn_java_version('1.6', MVN)
 
-def release_branch(release):
+# Utility that returns the name of the release branch for a given version
-  return 'release_branch_%s' % (release)
+def release_branch(version):
+  return 'release_branch_%s' % version
 
+# Creates a new release branch from the given source branch
+# and rebases the source branch from the remote before creating
+# the release branch. Note: This fails if the source branch
+# doesn't exist on the provided remote.
 def create_release_branch(remote, src_branch, release):
-  run('git checkout %s' % (src_branch))
+  run('git checkout %s' % src_branch)
   run('git pull --rebase %s %s' % (remote, src_branch))
   run('git checkout -b %s' % (release_branch(release)))
 
 
+# Reads the given file and applies the
+# callback to it. If the callback changed
+# a line the given file is replaced with
+# the modified input.
 def process_file(file_path, line_callback):
   fh, abs_path = tempfile.mkstemp()
+  modified = False
   with open(abs_path,'w') as new_file:
     with open(file_path) as old_file:
       for line in old_file:
-        new_file.write(line_callback(line))
+        new_line = line_callback(line)
+        modified = modified or (new_line != line)
+        new_file.write(new_line)
   os.close(fh)
-  #Remove original file
+  if modified:
-  os.remove(file_path)
+    #Remove original file
-  #Move new file
+    os.remove(file_path)
-  shutil.move(abs_path, file_path)
+    #Move new file
-  
+    shutil.move(abs_path, file_path)
+    return True
+  else:
+    # nothing to do - just remove the tmp file
+    os.remove(abs_path)
+    return False
 
+# Walks the given directory path (defaults to 'docs')
+# and replaces all 'coming[$version]' tags with
+# 'added[$version]'. This method only accesses asciidoc files.
+def update_reference_docs(release_version, path='docs'):
+  pattern = 'coming[%s' % (release_version)
+  replacement = 'added[%s' % (release_version)
+  pending_files = []
+  def callback(line):
+    return line.replace(pattern, replacement)
+  for root, _, file_names in os.walk(path):
+    for file_name in fnmatch.filter(file_names, '*.asciidoc'):
+      full_path = os.path.join(root, file_name)
+      if process_file(full_path, callback):
+        pending_files.append(os.path.join(root, file_name))
+  return pending_files
+
+# Moves the pom.xml file from a snapshot to a release
 def remove_maven_snapshot(pom, release):
   pattern = '<version>%s-SNAPSHOT</version>' % (release)
   replacement = '<version>%s</version>' % (release)
@@ -138,7 +177,7 @@ def remove_maven_snapshot(pom, release):
     return line.replace(pattern, replacement)
   process_file(pom, callback)
 
-
+# Moves the Version.java file from a snapshot to a release
 def remove_version_snapshot(version_file, release):
   # 1.0.0.Beta1 -> 1_0_0_Beat1
   release = release.replace('.', '_')
@@ -148,10 +187,12 @@ def remove_version_snapshot(version_file, release):
     return line.replace(pattern, replacement)
   process_file(version_file, callback)
 
+# Stages the given files for the next git commit
 def add_pending_files(*files):
   for file in files:
     run('git add %s' % (file))
 
+# Executes a git commit with 'release [version]' as the commit message
 def commit_release(release):
   run('git commit -m "release [%s]"' % release)
 
@@ -161,6 +202,7 @@ def tag_release(release):
 def run_mvn(*cmd):
   for c in cmd:
     run('%s; %s %s' % (java_exe(), MVN, c))
+
 def build_release(run_tests=False, dry_run=True, cpus=1):
   target = 'deploy'
   if dry_run:
@@ -176,28 +218,38 @@ def wait_for_node_startup(host='127.0.0.1', port=9200,timeout=15):
   conn = HTTPConnection(host, port, timeout);
   for _ in range(timeout):
     try:
+      log('Waiting until node becomes available for 1 second')
       time.sleep(1)
+      log('Check if node is available')
       conn.request('GET', '')
       res = conn.getresponse()
       if res.status == 200:
         return True
-    except:
+    except Exception as e:
-      pass #that is ok it might not be there yet
+      log("Failed while waiting for node - Exception: [%s]" % (e.message))
+      #that is ok it might not be there yet
 
   return False
 
+# Checks the pom.xml for the release version.
+# This method fails if the pom file has no SNAPSHOT version set ie.
+# if the version is already on a release version we fail.
+# Returns the next version string ie. 0.90.7
 def find_release_version(src_branch):
-  run('git checkout %s' % (src_branch))
+  run('git checkout %s' % src_branch)
   with open('pom.xml') as file:
     for line in file:
       match = re.search(r'<version>(.+)-SNAPSHOT</version>', line)
       if match:
         return match.group(1)
-    raise RuntimeError('Could not find release version in branch %s' % (src_branch))
+    raise RuntimeError('Could not find release version in branch %s' % src_branch)
 
 def get_artifacts(release, path='target/releases/'):
   return [os.path.join(path, 'elasticsearch-%s.%s' % (release, t)) for t in ['deb', 'tar.gz', 'zip']]
 
+# Generates sha1 checsums for all files
+# and returns the checksum files as well
+# as the given files in a list
 def generate_checksums(files):
   res = []
   for release_file in files:
@@ -351,12 +403,14 @@ if __name__ == '__main__':
   print('  Created release branch [%s]' % (release_branch(release_version)))
   success = False
   try:
+    pending_files = [POM_FILE, VERSION_FILE]
     remove_maven_snapshot(POM_FILE, release_version)
     remove_version_snapshot(VERSION_FILE, release_version)
+    pending_files = pending_files + update_reference_docs(release_version)
     print('  Done removing snapshot version')
-    add_pending_files(VERSION_FILE, POM_FILE)
+    add_pending_files(*pending_files) # expects var args use * to expand
     commit_release(release_version)
-    print('  Committed release version [%s]' % (release_version))
+    print('  Committed release version [%s]' % release_version)
     print(''.join(['-' for _ in range(80)]))
     print('Building Release candidate')
     input('Press Enter to continue...')