Release script: Improve automation for package repositories

* Automatic package repository creation for debian and rpm repositories using deb-s3 and rpm-s3 tools
* Fixing paths in email for repositories
* Add manual verification step for maven staging repo
* Do not create release directory in /tmp, because we might loose it on VMs
* Removed unused download-s3 script
* Add signage check for RPM
* Removed download-s3.py/upload-s3.py, as they are unused

Closes #13209

dev-tools/prepare_release_candidate.py

@@ -34,6 +34,8 @@ import tempfile
 import re
 import os
 import shutil
+from functools import partial
+import sys
 
 VERSION_FILE = 'core/src/main/java/org/elasticsearch/Version.java'
 POM_FILE = 'pom.xml'
@@ -61,7 +63,7 @@ To install the deb from an APT repo:
 
 APT line sources.list line:
 
-deb http://download.elasticsearch.org/elasticsearch/staging/%(version)s-%(hash)s/repos/elasticsearch/%(major_minor_version)s/debian/ stable main
+deb http://download.elasticsearch.org/elasticsearch/staging/%(version)s-%(hash)s/repos/%(major_minor_version)s/debian/ stable main
 
 To install the RPM, create a YUM file like:
 
@@ -71,7 +73,7 @@ containing:
 
 [elasticsearch-2.0]
 name=Elasticsearch repository for packages
-baseurl=http://download.elasticsearch.org/elasticsearch/staging/%(version)s-%(hash)s/repos/elasticsearch/%(major_minor_version)s/centos
+baseurl=http://download.elasticsearch.org/elasticsearch/staging/%(version)s-%(hash)s/repos/%(major_minor_version)s/centos
 gpgcheck=1
 gpgkey=http://packages.elastic.co/GPG-KEY-elasticsearch
 enabled=1
@@ -175,6 +177,64 @@ def find_release_version():
         return match.group(1)
     raise RuntimeError('Could not find release version in branch')
 
+# Checks if the produced RPM is signed with the supplied GPG key
+def ensure_rpm_is_signed(rpm, gpg_key):
+  rpm_check_signature_cmd = 'rpm -v -K %s | grep -qi %s' % (rpm, gpg_key)
+  try:
+    subprocess.check_output(rpm_check_signature_cmd, shell=True)
+  except:
+    raise RuntimeError('Aborting. RPM does not seem to be signed, check with: rpm -v -K %s' % rpm)
+
+# Checks if a command exists, needed for external binaries
+def check_command_exists(name, cmd):
+  try:
+    subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT)
+  except subprocess.CalledProcessError:
+    raise RuntimeError('Could not run command %s - please make sure it is installed and in $PATH' % (name))
+
+# console colors
+COLOR_OK = '\033[92m'
+COLOR_END = '\033[0m'
+COLOR_FAIL = '\033[91m'
+def run_and_print(text, run_function):
+  try:
+    print(text, end='')
+    run_function()
+    print(COLOR_OK + 'OK' + COLOR_END)
+    return True
+  except RuntimeError:
+    print(COLOR_FAIL + 'NOT OK' + COLOR_END)
+    return False
+
+def check_env_var(text, env_var):
+  try:
+    print(text, end='')
+    os.environ[env_var]
+    print(COLOR_OK + 'OK' + COLOR_END)
+    return True
+  except KeyError:
+    print(COLOR_FAIL + 'NOT OK' + COLOR_END)
+    return False
+
+def check_environment_and_commandline_tools(check_only):
+  checks = list()
+  checks.append(check_env_var('Checking for AWS env configuration AWS_SECRET_ACCESS_KEY_ID...     ', 'AWS_SECRET_ACCESS_KEY'))
+  checks.append(check_env_var('Checking for AWS env configuration AWS_ACCESS_KEY_ID...            ', 'AWS_ACCESS_KEY_ID'))
+  checks.append(run_and_print('Checking command: rpm...            ', partial(check_command_exists, 'rpm', 'rpm --version')))
+  checks.append(run_and_print('Checking command: dpkg...           ', partial(check_command_exists, 'dpkg', 'dpkg --version')))
+  checks.append(run_and_print('Checking command: gpg...            ', partial(check_command_exists, 'gpg', 'gpg --version')))
+  checks.append(run_and_print('Checking command: expect...         ', partial(check_command_exists, 'expect', 'expect -v')))
+  checks.append(run_and_print('Checking command: createrepo...     ', partial(check_command_exists, 'createrepo', 'createrepo --version')))
+  checks.append(run_and_print('Checking command: s3cmd...          ', partial(check_command_exists, 's3cmd', 's3cmd --version')))
+  checks.append(run_and_print('Checking command: deb-s3...         ', partial(check_command_exists, 'deb-s3', 'deb-s3 -h')))
+  checks.append(run_and_print('Checking command: rpm-s3...         ', partial(check_command_exists, 'rpm-s3', 'rpm-s3 -h')))
+
+  if check_only:
+    sys.exit(0)
+
+  if False in checks:
+    print("Exiting due to failing checks")
+    sys.exit(0)
 
 if __name__ == "__main__":
   parser = argparse.ArgumentParser(description='Builds and publishes a Elasticsearch Release')
@@ -190,6 +250,8 @@ if __name__ == "__main__":
                       help='Allows you to specify a different gpg_key to be used instead of the default release key')
   parser.add_argument('--verbose', '-b', dest='verbose',
                       help='Runs the script in verbose mode')
+  parser.add_argument('--check-only', dest='check_only', action='store_true',
+                      help='Checks and reports for all requirements and then exits')
   parser.set_defaults(deploy=False)
   parser.set_defaults(skip_doc_check=False)
   parser.set_defaults(push=False)
@@ -203,6 +265,8 @@ if __name__ == "__main__":
   install_only = args.install_only
   VERBOSE = args.verbose
 
+  check_environment_and_commandline_tools(args.check_only)
+
   ensure_checkout_is_clean()
   release_version = find_release_version()
   if not re.match('(\d+\.\d+)\.*',release_version):
@@ -227,7 +291,9 @@ if __name__ == "__main__":
   print('*** Done removing snapshot version. DO NOT COMMIT THIS, WHEN CREATING A RELEASE CANDIDATE.')
 
   shortHash = subprocess.check_output('git log --pretty=format:"%h" -n 1', shell=True).decode('utf-8')
-  localRepo = '/tmp/elasticsearch-%s-%s' % (release_version, shortHash)
+  releaseDirectory = os.getenv('HOME') + '/elastic-releases'
+  os.mkdir(releaseDirectory)
+  localRepo = '%s/elasticsearch-%s-%s' % (releaseDirectory, release_version, shortHash)
   localRepoElasticsearch = localRepo + '/org/elasticsearch'
   if os.path.exists(localRepoElasticsearch):
     print('clean local repository %s' % localRepoElasticsearch)
@@ -240,8 +306,6 @@ if __name__ == "__main__":
   install_command = 'mvn clean %s -Prelease -Dskip.integ.tests=true -Dgpg.key="%s" -Dpackaging.rpm.rpmbuild=/usr/bin/rpmbuild -Drpm.sign=true -Dmaven.repo.local=%s -Dno.commit.pattern="\\bno(n|)commit\\b" -Dforbidden.test.signatures=""' % (mvn_target, gpg_key, localRepo)
   clean_repo_command = 'find %s -name _remote.repositories -exec rm {} \;' % (localRepoElasticsearch)
   rename_metadata_files_command = 'for i in $(find %s -name "maven-metadata-local.xml*") ; do mv "$i" "${i/-local/}" ; done' % (localRepoElasticsearch)
-  s3_sync_command = 's3cmd sync %s s3://download.elasticsearch.org/elasticsearch/staging/%s-%s/org/' % (localRepoElasticsearch, release_version, shortHash)
-  s3_bucket_sync_to = 'download.elasticsearch.org/elasticsearch/staging/%s-%s/repos' % (release_version, shortHash)
   if install_and_deploy:
     for cmd in [install_command, clean_repo_command]:
       run(cmd)
@@ -252,16 +316,44 @@ if __name__ == "__main__":
     print('  %s' % (install_command))
     print('  1. Remove all _remote.repositories: %s' % (clean_repo_command))
     print('  2. Rename all maven metadata files: %s' % (rename_metadata_files_command))
+
+  print('Ensuring that RPM has been signed')
+  rpm = '%s/distribution/rpm/elasticsearch/%s/elasticsearch-%s.rpm' % (localRepoElasticsearch, release_version, release_version)
+  ensure_rpm_is_signed(rpm, gpg_key)
+
+  # repository push commands
+  s3cmd_sync_to_staging_bucket_cmd = 's3cmd sync -P %s s3://download.elasticsearch.org/elasticsearch/staging/%s-%s/org/' % (localRepoElasticsearch, release_version, shortHash)
+  s3_bucket_sync_to = 'download.elasticsearch.org/elasticsearch/staging/%s-%s/repos' % (release_version, shortHash)
+  s3cmd_sync_official_repo_cmd = 's3cmd sync s3://packages.elasticsearch.org/elasticsearch/%s s3://%s' % (major_minor_version, s3_bucket_sync_to)
+
+  debs3_prefix = 'elasticsearch/staging/%s-%s/repos/debian' % (release_version, shortHash)
+  debs3_upload_cmd = 'deb-s3 upload --preserve-versions %s/distribution/deb/elasticsearch/%s/elasticsearch-%s.deb -b download.elasticsearch.org --prefix %s --sign %s --arch amd64' % (localRepoElasticsearch, release_version, release_version, prefix, gpg_key)
+  debs3_list_cmd = 'deb-s3 list -b download.elasticsearch.org --prefix %s' % (debs3_prefix)
+  debs3_verify_cmd = 'deb-s3 verify -b download.elasticsearch.org --prefix %s' % (debs3_prefix)
+  rpms3_prefix = 'elasticsearch/staging/%s-%s/repos/centos' % (release_version, shortHash)
+  rpms3_upload_cmd = 'rpm-s3 -v -b download.elasticsearch.org -p %s --sign --visibility public-read -k 0 %s' % (rpms3_prefix, rpm)
+
   if push:
-    run(s3_sync_command)
+    run(s3cmd_sync_to_staging_bucket_cmd)
-    print('Use rpm-s3/deb-s3 to push into repositories at %s' % s3_bucket_sync_to)
+    print('Syncing official package repository into staging s3 bucket')
+    run(s3cmd_sync_official_repo_cmd)
+    print('Uploading debian package (you will be prompted for the passphrase!)')
+    run(debs3_upload_cmd)
+    run(debs3_list_cmd)
+    run(debs3_verify_cmd)
+    print('Uploading rpm package (you will be prompted for the passphrase!)')
+    run(rpms3_upload_cmd)
   else:
     print('')
     print('*** To push a release candidate to s3 run: ')
     print('  1. Sync %s into S3 bucket' % (localRepoElasticsearch))
-    print ('    %s' % (s3_sync_command))
+    print ('    %s' % (s3cmd_sync_to_staging_bucket_cmd))
     print('  2. Create repositories: ')
-    print('     Use rpm-s3/deb-s3 to push into repositories at %s' % s3_bucket_sync_to)
+    print('     1. Sync existing repo into staging: %s' % s3cmd_sync_official_repo_cmd)
+    print('     2. Upload debian package (and sign it) %s' % debs3_upload_cmd)
+    print('     3. List all debian packages: %s' % debs3_list_cmd)
+    print('     4. Verify debian packages: %s' % debs3_verify_cmd)
+    print('     5. Upload RPM: %s' % rpms3_upload_cmd)
     print('')
     print('NOTE: the above mvn command will promt you several times for the GPG passphrase of the key you specified you can alternatively pass it via -Dgpg.passphrase=yourPassPhrase')
     print(' since RPM signing doesn\'t support gpg-agents the recommended way to set the password is to add a release profile to your settings.xml:')
@@ -280,8 +372,13 @@ if __name__ == "__main__":
   string_format_dict = {'version' : release_version, 'hash': shortHash, 'major_minor_version' : major_minor_version}
   print(MAIL_TEMPLATE % string_format_dict)
 
+  print('')
+  print('You can verify that pushing to the staging repository pushed all the artifacts by running (log into sonatype to find out the correct id):')
+  print(' python3 -B dev-tools/validate-maven-repository.py %s https://oss.sonatype.org/service/local/repositories/orgelasticsearch-IDTOFINDOUT/content/org/elasticsearch ' %(localElasticsearchRepo))
+
+  print('')
   print('To publish the release and the repo on S3 execute the following commands:')
-  print('   s3cmd cp --recursive s3://download.elasticsearch.org/elasticsearch/staging/%(version)s-%(hash)s/repos/elasticsearch/%(major_minor_version)s/ s3://packages.elasticsearch.org/elasticsearch/%(major_minor_version)s'  % string_format_dict)
+  print('   s3cmd cp --recursive s3://download.elasticsearch.org/elasticsearch/staging/%(version)s-%(hash)s/repos/%(major_minor_version)s/ s3://packages.elasticsearch.org/elasticsearch/%(major_minor_version)s'  % string_format_dict)
   print('   s3cmd cp --recursive s3://download.elasticsearch.org/elasticsearch/staging/%(version)s-%(hash)s/org/ s3://download.elasticsearch.org/elasticsearch/release/org'  % string_format_dict)
   print('Now go ahead and tag the release:')
   print('   git tag -a v%(version)s %(hash)s'  % string_format_dict)