Release script: Improve automation for package repositories
* Automatic package repository creation for debian and rpm repositories using deb-s3 and rpm-s3 tools * Fixing paths in email for repositories * Add manual verification step for maven staging repo * Do not create release directory in /tmp, because we might loose it on VMs * Removed unused download-s3 script * Add signage check for RPM * Removed download-s3.py/upload-s3.py, as they are unused Closes #13209
This commit is contained in:
Alexander Reelsen
parent
9c91af9b7f
commit
5f6aeb982d
|
|
@ -34,6 +34,8 @@ import tempfile
|
|||
import re
|
||||
import os
|
||||
import shutil
|
||||
from functools import partial
|
||||
import sys
|
||||
|
||||
VERSION_FILE = 'core/src/main/java/org/elasticsearch/Version.java'
|
||||
POM_FILE = 'pom.xml'
|
||||
|
|
@ -61,7 +63,7 @@ To install the deb from an APT repo:
|
|||
|
||||
APT line sources.list line:
|
||||
|
||||
deb http://download.elasticsearch.org/elasticsearch/staging/%(version)s-%(hash)s/repos/elasticsearch/%(major_minor_version)s/debian/ stable main
|
||||
deb http://download.elasticsearch.org/elasticsearch/staging/%(version)s-%(hash)s/repos/%(major_minor_version)s/debian/ stable main
|
||||
|
||||
To install the RPM, create a YUM file like:
|
||||
|
||||
|
|
@ -71,7 +73,7 @@ containing:
|
|||
|
||||
[elasticsearch-2.0]
|
||||
name=Elasticsearch repository for packages
|
||||
baseurl=http://download.elasticsearch.org/elasticsearch/staging/%(version)s-%(hash)s/repos/elasticsearch/%(major_minor_version)s/centos
|
||||
baseurl=http://download.elasticsearch.org/elasticsearch/staging/%(version)s-%(hash)s/repos/%(major_minor_version)s/centos
|
||||
gpgcheck=1
|
||||
gpgkey=http://packages.elastic.co/GPG-KEY-elasticsearch
|
||||
enabled=1
|
||||
|
|
@ -175,6 +177,64 @@ def find_release_version():
|
|||
return match.group(1)
|
||||
raise RuntimeError('Could not find release version in branch')
|
||||
|
||||
# Checks if the produced RPM is signed with the supplied GPG key
|
||||
def ensure_rpm_is_signed(rpm, gpg_key):
|
||||
rpm_check_signature_cmd = 'rpm -v -K %s | grep -qi %s' % (rpm, gpg_key)
|
||||
try:
|
||||
subprocess.check_output(rpm_check_signature_cmd, shell=True)
|
||||
except:
|
||||
raise RuntimeError('Aborting. RPM does not seem to be signed, check with: rpm -v -K %s' % rpm)
|
||||
|
||||
# Checks if a command exists, needed for external binaries
|
||||
def check_command_exists(name, cmd):
|
||||
try:
|
||||
subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT)
|
||||
except subprocess.CalledProcessError:
|
||||
raise RuntimeError('Could not run command %s - please make sure it is installed and in $PATH' % (name))
|
||||
|
||||
# console colors
|
||||
COLOR_OK = '\033[92m'
|
||||
COLOR_END = '\033[0m'
|
||||
COLOR_FAIL = '\033[91m'
|
||||
def run_and_print(text, run_function):
|
||||
try:
|
||||
print(text, end='')
|
||||
run_function()
|
||||
print(COLOR_OK + 'OK' + COLOR_END)
|
||||
return True
|
||||
except RuntimeError:
|
||||
print(COLOR_FAIL + 'NOT OK' + COLOR_END)
|
||||
return False
|
||||
|
||||
def check_env_var(text, env_var):
|
||||
try:
|
||||
print(text, end='')
|
||||
os.environ[env_var]
|
||||
print(COLOR_OK + 'OK' + COLOR_END)
|
||||
return True
|
||||
except KeyError:
|
||||
print(COLOR_FAIL + 'NOT OK' + COLOR_END)
|
||||
return False
|
||||
|
||||
def check_environment_and_commandline_tools(check_only):
|
||||
checks = list()
|
||||
checks.append(check_env_var('Checking for AWS env configuration AWS_SECRET_ACCESS_KEY_ID... ', 'AWS_SECRET_ACCESS_KEY'))
|
||||
checks.append(check_env_var('Checking for AWS env configuration AWS_ACCESS_KEY_ID... ', 'AWS_ACCESS_KEY_ID'))
|
||||
checks.append(run_and_print('Checking command: rpm... ', partial(check_command_exists, 'rpm', 'rpm --version')))
|
||||
checks.append(run_and_print('Checking command: dpkg... ', partial(check_command_exists, 'dpkg', 'dpkg --version')))
|
||||
checks.append(run_and_print('Checking command: gpg... ', partial(check_command_exists, 'gpg', 'gpg --version')))
|
||||
checks.append(run_and_print('Checking command: expect... ', partial(check_command_exists, 'expect', 'expect -v')))
|
||||
checks.append(run_and_print('Checking command: createrepo... ', partial(check_command_exists, 'createrepo', 'createrepo --version')))
|
||||
checks.append(run_and_print('Checking command: s3cmd... ', partial(check_command_exists, 's3cmd', 's3cmd --version')))
|
||||
checks.append(run_and_print('Checking command: deb-s3... ', partial(check_command_exists, 'deb-s3', 'deb-s3 -h')))
|
||||
checks.append(run_and_print('Checking command: rpm-s3... ', partial(check_command_exists, 'rpm-s3', 'rpm-s3 -h')))
|
||||
|
||||
if check_only:
|
||||
sys.exit(0)
|
||||
|
||||
if False in checks:
|
||||
print("Exiting due to failing checks")
|
||||
sys.exit(0)
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description='Builds and publishes a Elasticsearch Release')
|
||||
|
|
@ -190,6 +250,8 @@ if __name__ == "__main__":
|
|||
help='Allows you to specify a different gpg_key to be used instead of the default release key')
|
||||
parser.add_argument('--verbose', '-b', dest='verbose',
|
||||
help='Runs the script in verbose mode')
|
||||
parser.add_argument('--check-only', dest='check_only', action='store_true',
|
||||
help='Checks and reports for all requirements and then exits')
|
||||
parser.set_defaults(deploy=False)
|
||||
parser.set_defaults(skip_doc_check=False)
|
||||
parser.set_defaults(push=False)
|
||||
|
|
@ -203,6 +265,8 @@ if __name__ == "__main__":
|
|||
install_only = args.install_only
|
||||
VERBOSE = args.verbose
|
||||
|
||||
check_environment_and_commandline_tools(args.check_only)
|
||||
|
||||
ensure_checkout_is_clean()
|
||||
release_version = find_release_version()
|
||||
if not re.match('(\d+\.\d+)\.*',release_version):
|
||||
|
|
@ -227,7 +291,9 @@ if __name__ == "__main__":
|
|||
print('*** Done removing snapshot version. DO NOT COMMIT THIS, WHEN CREATING A RELEASE CANDIDATE.')
|
||||
|
||||
shortHash = subprocess.check_output('git log --pretty=format:"%h" -n 1', shell=True).decode('utf-8')
|
||||
localRepo = '/tmp/elasticsearch-%s-%s' % (release_version, shortHash)
|
||||
releaseDirectory = os.getenv('HOME') + '/elastic-releases'
|
||||
os.mkdir(releaseDirectory)
|
||||
localRepo = '%s/elasticsearch-%s-%s' % (releaseDirectory, release_version, shortHash)
|
||||
localRepoElasticsearch = localRepo + '/org/elasticsearch'
|
||||
if os.path.exists(localRepoElasticsearch):
|
||||
print('clean local repository %s' % localRepoElasticsearch)
|
||||
|
|
@ -240,8 +306,6 @@ if __name__ == "__main__":
|
|||
install_command = 'mvn clean %s -Prelease -Dskip.integ.tests=true -Dgpg.key="%s" -Dpackaging.rpm.rpmbuild=/usr/bin/rpmbuild -Drpm.sign=true -Dmaven.repo.local=%s -Dno.commit.pattern="\\bno(n|)commit\\b" -Dforbidden.test.signatures=""' % (mvn_target, gpg_key, localRepo)
|
||||
clean_repo_command = 'find %s -name _remote.repositories -exec rm {} \;' % (localRepoElasticsearch)
|
||||
rename_metadata_files_command = 'for i in $(find %s -name "maven-metadata-local.xml*") ; do mv "$i" "${i/-local/}" ; done' % (localRepoElasticsearch)
|
||||
s3_sync_command = 's3cmd sync %s s3://download.elasticsearch.org/elasticsearch/staging/%s-%s/org/' % (localRepoElasticsearch, release_version, shortHash)
|
||||
s3_bucket_sync_to = 'download.elasticsearch.org/elasticsearch/staging/%s-%s/repos' % (release_version, shortHash)
|
||||
if install_and_deploy:
|
||||
for cmd in [install_command, clean_repo_command]:
|
||||
run(cmd)
|
||||
|
|
@ -252,16 +316,44 @@ if __name__ == "__main__":
|
|||
print(' %s' % (install_command))
|
||||
print(' 1. Remove all _remote.repositories: %s' % (clean_repo_command))
|
||||
print(' 2. Rename all maven metadata files: %s' % (rename_metadata_files_command))
|
||||
|
||||
print('Ensuring that RPM has been signed')
|
||||
rpm = '%s/distribution/rpm/elasticsearch/%s/elasticsearch-%s.rpm' % (localRepoElasticsearch, release_version, release_version)
|
||||
ensure_rpm_is_signed(rpm, gpg_key)
|
||||
|
||||
# repository push commands
|
||||
s3cmd_sync_to_staging_bucket_cmd = 's3cmd sync -P %s s3://download.elasticsearch.org/elasticsearch/staging/%s-%s/org/' % (localRepoElasticsearch, release_version, shortHash)
|
||||
s3_bucket_sync_to = 'download.elasticsearch.org/elasticsearch/staging/%s-%s/repos' % (release_version, shortHash)
|
||||
s3cmd_sync_official_repo_cmd = 's3cmd sync s3://packages.elasticsearch.org/elasticsearch/%s s3://%s' % (major_minor_version, s3_bucket_sync_to)
|
||||
|
||||
debs3_prefix = 'elasticsearch/staging/%s-%s/repos/debian' % (release_version, shortHash)
|
||||
debs3_upload_cmd = 'deb-s3 upload --preserve-versions %s/distribution/deb/elasticsearch/%s/elasticsearch-%s.deb -b download.elasticsearch.org --prefix %s --sign %s --arch amd64' % (localRepoElasticsearch, release_version, release_version, prefix, gpg_key)
|
||||
debs3_list_cmd = 'deb-s3 list -b download.elasticsearch.org --prefix %s' % (debs3_prefix)
|
||||
debs3_verify_cmd = 'deb-s3 verify -b download.elasticsearch.org --prefix %s' % (debs3_prefix)
|
||||
rpms3_prefix = 'elasticsearch/staging/%s-%s/repos/centos' % (release_version, shortHash)
|
||||
rpms3_upload_cmd = 'rpm-s3 -v -b download.elasticsearch.org -p %s --sign --visibility public-read -k 0 %s' % (rpms3_prefix, rpm)
|
||||
|
||||
if push:
|
||||
run(s3_sync_command)
|
||||
print('Use rpm-s3/deb-s3 to push into repositories at %s' % s3_bucket_sync_to)
|
||||
run(s3cmd_sync_to_staging_bucket_cmd)
|
||||
print('Syncing official package repository into staging s3 bucket')
|
||||
run(s3cmd_sync_official_repo_cmd)
|
||||
print('Uploading debian package (you will be prompted for the passphrase!)')
|
||||
run(debs3_upload_cmd)
|
||||
run(debs3_list_cmd)
|
||||
run(debs3_verify_cmd)
|
||||
print('Uploading rpm package (you will be prompted for the passphrase!)')
|
||||
run(rpms3_upload_cmd)
|
||||
else:
|
||||
print('')
|
||||
print('*** To push a release candidate to s3 run: ')
|
||||
print(' 1. Sync %s into S3 bucket' % (localRepoElasticsearch))
|
||||
print (' %s' % (s3_sync_command))
|
||||
print (' %s' % (s3cmd_sync_to_staging_bucket_cmd))
|
||||
print(' 2. Create repositories: ')
|
||||
print(' Use rpm-s3/deb-s3 to push into repositories at %s' % s3_bucket_sync_to)
|
||||
print(' 1. Sync existing repo into staging: %s' % s3cmd_sync_official_repo_cmd)
|
||||
print(' 2. Upload debian package (and sign it) %s' % debs3_upload_cmd)
|
||||
print(' 3. List all debian packages: %s' % debs3_list_cmd)
|
||||
print(' 4. Verify debian packages: %s' % debs3_verify_cmd)
|
||||
print(' 5. Upload RPM: %s' % rpms3_upload_cmd)
|
||||
print('')
|
||||
print('NOTE: the above mvn command will promt you several times for the GPG passphrase of the key you specified you can alternatively pass it via -Dgpg.passphrase=yourPassPhrase')
|
||||
print(' since RPM signing doesn\'t support gpg-agents the recommended way to set the password is to add a release profile to your settings.xml:')
|
||||
|
|
@ -280,8 +372,13 @@ if __name__ == "__main__":
|
|||
string_format_dict = {'version' : release_version, 'hash': shortHash, 'major_minor_version' : major_minor_version}
|
||||
print(MAIL_TEMPLATE % string_format_dict)
|
||||
|
||||
print('')
|
||||
print('You can verify that pushing to the staging repository pushed all the artifacts by running (log into sonatype to find out the correct id):')
|
||||
print(' python3 -B dev-tools/validate-maven-repository.py %s https://oss.sonatype.org/service/local/repositories/orgelasticsearch-IDTOFINDOUT/content/org/elasticsearch ' %(localElasticsearchRepo))
|
||||
|
||||
print('')
|
||||
print('To publish the release and the repo on S3 execute the following commands:')
|
||||
print(' s3cmd cp --recursive s3://download.elasticsearch.org/elasticsearch/staging/%(version)s-%(hash)s/repos/elasticsearch/%(major_minor_version)s/ s3://packages.elasticsearch.org/elasticsearch/%(major_minor_version)s' % string_format_dict)
|
||||
print(' s3cmd cp --recursive s3://download.elasticsearch.org/elasticsearch/staging/%(version)s-%(hash)s/repos/%(major_minor_version)s/ s3://packages.elasticsearch.org/elasticsearch/%(major_minor_version)s' % string_format_dict)
|
||||
print(' s3cmd cp --recursive s3://download.elasticsearch.org/elasticsearch/staging/%(version)s-%(hash)s/org/ s3://download.elasticsearch.org/elasticsearch/release/org' % string_format_dict)
|
||||
print('Now go ahead and tag the release:')
|
||||
print(' git tag -a v%(version)s %(hash)s' % string_format_dict)
|
||||
|
|
|
|||
Loading…
Reference in New Issue