diff --git a/dev-tools/prepare_release_candidate.py b/dev-tools/prepare_release_candidate.py index f99446f5bee..51387cc6d89 100644 --- a/dev-tools/prepare_release_candidate.py +++ b/dev-tools/prepare_release_candidate.py @@ -34,6 +34,8 @@ import tempfile import re import os import shutil +from functools import partial +import sys VERSION_FILE = 'core/src/main/java/org/elasticsearch/Version.java' POM_FILE = 'pom.xml' @@ -61,7 +63,7 @@ To install the deb from an APT repo: APT line sources.list line: -deb http://download.elasticsearch.org/elasticsearch/staging/%(version)s-%(hash)s/repos/elasticsearch/%(major_minor_version)s/debian/ stable main +deb http://download.elasticsearch.org/elasticsearch/staging/%(version)s-%(hash)s/repos/%(major_minor_version)s/debian/ stable main To install the RPM, create a YUM file like: @@ -71,7 +73,7 @@ containing: [elasticsearch-2.0] name=Elasticsearch repository for packages -baseurl=http://download.elasticsearch.org/elasticsearch/staging/%(version)s-%(hash)s/repos/elasticsearch/%(major_minor_version)s/centos +baseurl=http://download.elasticsearch.org/elasticsearch/staging/%(version)s-%(hash)s/repos/%(major_minor_version)s/centos gpgcheck=1 gpgkey=http://packages.elastic.co/GPG-KEY-elasticsearch enabled=1 @@ -175,6 +177,64 @@ def find_release_version(): return match.group(1) raise RuntimeError('Could not find release version in branch') +# Checks if the produced RPM is signed with the supplied GPG key +def ensure_rpm_is_signed(rpm, gpg_key): + rpm_check_signature_cmd = 'rpm -v -K %s | grep -qi %s' % (rpm, gpg_key) + try: + subprocess.check_output(rpm_check_signature_cmd, shell=True) + except: + raise RuntimeError('Aborting. RPM does not seem to be signed, check with: rpm -v -K %s' % rpm) + +# Checks if a command exists, needed for external binaries +def check_command_exists(name, cmd): + try: + subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT) + except subprocess.CalledProcessError: + raise RuntimeError('Could not run command %s - please make sure it is installed and in $PATH' % (name)) + +# console colors +COLOR_OK = '\033[92m' +COLOR_END = '\033[0m' +COLOR_FAIL = '\033[91m' +def run_and_print(text, run_function): + try: + print(text, end='') + run_function() + print(COLOR_OK + 'OK' + COLOR_END) + return True + except RuntimeError: + print(COLOR_FAIL + 'NOT OK' + COLOR_END) + return False + +def check_env_var(text, env_var): + try: + print(text, end='') + os.environ[env_var] + print(COLOR_OK + 'OK' + COLOR_END) + return True + except KeyError: + print(COLOR_FAIL + 'NOT OK' + COLOR_END) + return False + +def check_environment_and_commandline_tools(check_only): + checks = list() + checks.append(check_env_var('Checking for AWS env configuration AWS_SECRET_ACCESS_KEY_ID... ', 'AWS_SECRET_ACCESS_KEY')) + checks.append(check_env_var('Checking for AWS env configuration AWS_ACCESS_KEY_ID... ', 'AWS_ACCESS_KEY_ID')) + checks.append(run_and_print('Checking command: rpm... ', partial(check_command_exists, 'rpm', 'rpm --version'))) + checks.append(run_and_print('Checking command: dpkg... ', partial(check_command_exists, 'dpkg', 'dpkg --version'))) + checks.append(run_and_print('Checking command: gpg... ', partial(check_command_exists, 'gpg', 'gpg --version'))) + checks.append(run_and_print('Checking command: expect... ', partial(check_command_exists, 'expect', 'expect -v'))) + checks.append(run_and_print('Checking command: createrepo... ', partial(check_command_exists, 'createrepo', 'createrepo --version'))) + checks.append(run_and_print('Checking command: s3cmd... ', partial(check_command_exists, 's3cmd', 's3cmd --version'))) + checks.append(run_and_print('Checking command: deb-s3... ', partial(check_command_exists, 'deb-s3', 'deb-s3 -h'))) + checks.append(run_and_print('Checking command: rpm-s3... ', partial(check_command_exists, 'rpm-s3', 'rpm-s3 -h'))) + + if check_only: + sys.exit(0) + + if False in checks: + print("Exiting due to failing checks") + sys.exit(0) if __name__ == "__main__": parser = argparse.ArgumentParser(description='Builds and publishes a Elasticsearch Release') @@ -190,6 +250,8 @@ if __name__ == "__main__": help='Allows you to specify a different gpg_key to be used instead of the default release key') parser.add_argument('--verbose', '-b', dest='verbose', help='Runs the script in verbose mode') + parser.add_argument('--check-only', dest='check_only', action='store_true', + help='Checks and reports for all requirements and then exits') parser.set_defaults(deploy=False) parser.set_defaults(skip_doc_check=False) parser.set_defaults(push=False) @@ -203,6 +265,8 @@ if __name__ == "__main__": install_only = args.install_only VERBOSE = args.verbose + check_environment_and_commandline_tools(args.check_only) + ensure_checkout_is_clean() release_version = find_release_version() if not re.match('(\d+\.\d+)\.*',release_version): @@ -227,7 +291,9 @@ if __name__ == "__main__": print('*** Done removing snapshot version. DO NOT COMMIT THIS, WHEN CREATING A RELEASE CANDIDATE.') shortHash = subprocess.check_output('git log --pretty=format:"%h" -n 1', shell=True).decode('utf-8') - localRepo = '/tmp/elasticsearch-%s-%s' % (release_version, shortHash) + releaseDirectory = os.getenv('HOME') + '/elastic-releases' + os.mkdir(releaseDirectory) + localRepo = '%s/elasticsearch-%s-%s' % (releaseDirectory, release_version, shortHash) localRepoElasticsearch = localRepo + '/org/elasticsearch' if os.path.exists(localRepoElasticsearch): print('clean local repository %s' % localRepoElasticsearch) @@ -240,8 +306,6 @@ if __name__ == "__main__": install_command = 'mvn clean %s -Prelease -Dskip.integ.tests=true -Dgpg.key="%s" -Dpackaging.rpm.rpmbuild=/usr/bin/rpmbuild -Drpm.sign=true -Dmaven.repo.local=%s -Dno.commit.pattern="\\bno(n|)commit\\b" -Dforbidden.test.signatures=""' % (mvn_target, gpg_key, localRepo) clean_repo_command = 'find %s -name _remote.repositories -exec rm {} \;' % (localRepoElasticsearch) rename_metadata_files_command = 'for i in $(find %s -name "maven-metadata-local.xml*") ; do mv "$i" "${i/-local/}" ; done' % (localRepoElasticsearch) - s3_sync_command = 's3cmd sync %s s3://download.elasticsearch.org/elasticsearch/staging/%s-%s/org/' % (localRepoElasticsearch, release_version, shortHash) - s3_bucket_sync_to = 'download.elasticsearch.org/elasticsearch/staging/%s-%s/repos' % (release_version, shortHash) if install_and_deploy: for cmd in [install_command, clean_repo_command]: run(cmd) @@ -252,16 +316,44 @@ if __name__ == "__main__": print(' %s' % (install_command)) print(' 1. Remove all _remote.repositories: %s' % (clean_repo_command)) print(' 2. Rename all maven metadata files: %s' % (rename_metadata_files_command)) + + print('Ensuring that RPM has been signed') + rpm = '%s/distribution/rpm/elasticsearch/%s/elasticsearch-%s.rpm' % (localRepoElasticsearch, release_version, release_version) + ensure_rpm_is_signed(rpm, gpg_key) + + # repository push commands + s3cmd_sync_to_staging_bucket_cmd = 's3cmd sync -P %s s3://download.elasticsearch.org/elasticsearch/staging/%s-%s/org/' % (localRepoElasticsearch, release_version, shortHash) + s3_bucket_sync_to = 'download.elasticsearch.org/elasticsearch/staging/%s-%s/repos' % (release_version, shortHash) + s3cmd_sync_official_repo_cmd = 's3cmd sync s3://packages.elasticsearch.org/elasticsearch/%s s3://%s' % (major_minor_version, s3_bucket_sync_to) + + debs3_prefix = 'elasticsearch/staging/%s-%s/repos/debian' % (release_version, shortHash) + debs3_upload_cmd = 'deb-s3 upload --preserve-versions %s/distribution/deb/elasticsearch/%s/elasticsearch-%s.deb -b download.elasticsearch.org --prefix %s --sign %s --arch amd64' % (localRepoElasticsearch, release_version, release_version, prefix, gpg_key) + debs3_list_cmd = 'deb-s3 list -b download.elasticsearch.org --prefix %s' % (debs3_prefix) + debs3_verify_cmd = 'deb-s3 verify -b download.elasticsearch.org --prefix %s' % (debs3_prefix) + rpms3_prefix = 'elasticsearch/staging/%s-%s/repos/centos' % (release_version, shortHash) + rpms3_upload_cmd = 'rpm-s3 -v -b download.elasticsearch.org -p %s --sign --visibility public-read -k 0 %s' % (rpms3_prefix, rpm) + if push: - run(s3_sync_command) - print('Use rpm-s3/deb-s3 to push into repositories at %s' % s3_bucket_sync_to) + run(s3cmd_sync_to_staging_bucket_cmd) + print('Syncing official package repository into staging s3 bucket') + run(s3cmd_sync_official_repo_cmd) + print('Uploading debian package (you will be prompted for the passphrase!)') + run(debs3_upload_cmd) + run(debs3_list_cmd) + run(debs3_verify_cmd) + print('Uploading rpm package (you will be prompted for the passphrase!)') + run(rpms3_upload_cmd) else: print('') print('*** To push a release candidate to s3 run: ') print(' 1. Sync %s into S3 bucket' % (localRepoElasticsearch)) - print (' %s' % (s3_sync_command)) + print (' %s' % (s3cmd_sync_to_staging_bucket_cmd)) print(' 2. Create repositories: ') - print(' Use rpm-s3/deb-s3 to push into repositories at %s' % s3_bucket_sync_to) + print(' 1. Sync existing repo into staging: %s' % s3cmd_sync_official_repo_cmd) + print(' 2. Upload debian package (and sign it) %s' % debs3_upload_cmd) + print(' 3. List all debian packages: %s' % debs3_list_cmd) + print(' 4. Verify debian packages: %s' % debs3_verify_cmd) + print(' 5. Upload RPM: %s' % rpms3_upload_cmd) print('') print('NOTE: the above mvn command will promt you several times for the GPG passphrase of the key you specified you can alternatively pass it via -Dgpg.passphrase=yourPassPhrase') print(' since RPM signing doesn\'t support gpg-agents the recommended way to set the password is to add a release profile to your settings.xml:') @@ -280,8 +372,13 @@ if __name__ == "__main__": string_format_dict = {'version' : release_version, 'hash': shortHash, 'major_minor_version' : major_minor_version} print(MAIL_TEMPLATE % string_format_dict) + print('') + print('You can verify that pushing to the staging repository pushed all the artifacts by running (log into sonatype to find out the correct id):') + print(' python3 -B dev-tools/validate-maven-repository.py %s https://oss.sonatype.org/service/local/repositories/orgelasticsearch-IDTOFINDOUT/content/org/elasticsearch ' %(localElasticsearchRepo)) + + print('') print('To publish the release and the repo on S3 execute the following commands:') - print(' s3cmd cp --recursive s3://download.elasticsearch.org/elasticsearch/staging/%(version)s-%(hash)s/repos/elasticsearch/%(major_minor_version)s/ s3://packages.elasticsearch.org/elasticsearch/%(major_minor_version)s' % string_format_dict) + print(' s3cmd cp --recursive s3://download.elasticsearch.org/elasticsearch/staging/%(version)s-%(hash)s/repos/%(major_minor_version)s/ s3://packages.elasticsearch.org/elasticsearch/%(major_minor_version)s' % string_format_dict) print(' s3cmd cp --recursive s3://download.elasticsearch.org/elasticsearch/staging/%(version)s-%(hash)s/org/ s3://download.elasticsearch.org/elasticsearch/release/org' % string_format_dict) print('Now go ahead and tag the release:') print(' git tag -a v%(version)s %(hash)s' % string_format_dict)