From 56cc87b7ccd41040d8fafcaf2da3680d96d10839 Mon Sep 17 00:00:00 2001 From: Alexander Reelsen Date: Fri, 23 Sep 2016 11:28:46 +0200 Subject: [PATCH] Smoke tester: Adjust to latest changes (#20611) * Removed the check for the correct hash in the `/` response, otherwise two hashes would be needed * Adapted to new URLs * Added two missing plugins * Downloaded gpg key only once --- dev-tools/smoke_test_rc.py | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/dev-tools/smoke_test_rc.py b/dev-tools/smoke_test_rc.py index 0a349851bd2..420ef35f36f 100644 --- a/dev-tools/smoke_test_rc.py +++ b/dev-tools/smoke_test_rc.py @@ -69,8 +69,10 @@ DEFAULT_PLUGINS = ["analysis-icu", "discovery-gce", "ingest-attachment", "ingest-geoip", + "ingest-user-agent", "lang-javascript", "lang-python", + "mapper-attachments", "mapper-murmur3", "mapper-size", "repository-azure", @@ -131,6 +133,13 @@ def download_and_verify(version, hash, files, base_url, plugins=DEFAULT_PLUGINS) try: downloaded_files = [] print(' ' + '*' * 80) + # here we create a temp gpg home where we download the release key as the only key into + # when we verify the signature it will fail if the signed key is not in the keystore and that + # way we keep the executing host unmodified since we don't have to import the key into the default keystore + gpg_home_dir = os.path.join(tmp_dir, "gpg_home_dir") + os.makedirs(gpg_home_dir, 0o700) + run('gpg --homedir %s --keyserver pool.sks-keyservers.net --recv-key D88E42B4' % gpg_home_dir) + for file in files: name = os.path.basename(file) print(' Smoketest file: %s' % name) @@ -139,7 +148,6 @@ def download_and_verify(version, hash, files, base_url, plugins=DEFAULT_PLUGINS) artifact_path = os.path.join(tmp_dir, file) downloaded_files.append(artifact_path) current_artifact_dir = os.path.dirname(artifact_path) - os.makedirs(current_artifact_dir) urllib.request.urlretrieve(url, os.path.join(tmp_dir, file)) sha1_url = ''.join([url, '.sha1']) checksum_file = artifact_path + ".sha1" @@ -155,12 +163,6 @@ def download_and_verify(version, hash, files, base_url, plugins=DEFAULT_PLUGINS) print(' Downloading %s' % (gpg_url)) urllib.request.urlretrieve(gpg_url, gpg_file) print(' Verifying gpg signature %s' % (gpg_file)) - # here we create a temp gpg home where we download the release key as the only key into - # when we verify the signature it will fail if the signed key is not in the keystore and that - # way we keep the executing host unmodified since we don't have to import the key into the default keystore - gpg_home_dir = os.path.join(current_artifact_dir, "gpg_home_dir") - os.makedirs(gpg_home_dir, 0o700) - run('gpg --homedir %s --keyserver pool.sks-keyservers.net --recv-key D88E42B4' % gpg_home_dir) run('cd %s && gpg --homedir %s --verify %s' % (current_artifact_dir, gpg_home_dir, os.path.basename(gpg_file))) print(' ' + '*' * 80) print() @@ -172,7 +174,7 @@ def download_and_verify(version, hash, files, base_url, plugins=DEFAULT_PLUGINS) def get_host_from_ports_file(es_dir): return read_fully(os.path.join(es_dir, 'logs/http.ports')).splitlines()[0] -def smoke_test_release(release, files, expected_hash, plugins): +def smoke_test_release(release, files, hash, plugins): for release_file in files: if not os.path.isfile(release_file): raise RuntimeError('Smoketest failed missing file %s' % (release_file)) @@ -191,7 +193,7 @@ def smoke_test_release(release, files, expected_hash, plugins): plugin_names = {} for plugin in plugins: print(' Install plugin [%s]' % (plugin)) - run('%s; export ES_JAVA_OPTS="-Des.plugins.staging=%s"; %s %s %s' % (java_exe(), expected_hash, es_plugin_path, 'install -b', plugin)) + run('%s; export ES_JAVA_OPTS="-Des.plugins.staging=%s"; %s %s %s' % (java_exe(), hash, es_plugin_path, 'install -b', plugin)) plugin_names[plugin] = True if 'x-pack' in plugin_names: headers = { 'Authorization' : 'Basic %s' % base64.b64encode(b"es_admin:foobar").decode("UTF-8") } @@ -222,8 +224,6 @@ def smoke_test_release(release, files, expected_hash, plugins): raise RuntimeError('Expected version [%s] but was [%s]' % (release, version['number'])) if version['build_snapshot']: raise RuntimeError('Expected non snapshot version') - if expected_hash != version['build_hash'].strip(): - raise RuntimeError('HEAD hash does not match expected [%s] but got [%s]' % (expected_hash, version['build_hash'])) print(' Verify if plugins are listed in _nodes') conn.request('GET', '/_nodes?plugin=true&pretty=true', headers=headers) res = conn.getresponse() @@ -262,7 +262,7 @@ if __name__ == "__main__": parser.add_argument('--version', '-v', dest='version', default=None, help='The Elasticsearch Version to smoke-tests', required=True) parser.add_argument('--hash', '-s', dest='hash', default=None, required=True, - help='The sha1 short hash of the git commit to smoketest') + help='The hash of the unified release') parser.add_argument('--plugins', '-p', dest='plugins', default=[], required=False, type=parse_list, help='A list of additional plugins to smoketest') parser.add_argument('--fetch_url', '-u', dest='url', default=None, @@ -277,16 +277,16 @@ if __name__ == "__main__": hash = args.hash url = args.url files = [ x % {'version': version} for x in [ - 'org/elasticsearch/distribution/tar/elasticsearch/%(version)s/elasticsearch-%(version)s.tar.gz', - 'org/elasticsearch/distribution/zip/elasticsearch/%(version)s/elasticsearch-%(version)s.zip', - 'org/elasticsearch/distribution/deb/elasticsearch/%(version)s/elasticsearch-%(version)s.deb', - 'org/elasticsearch/distribution/rpm/elasticsearch/%(version)s/elasticsearch-%(version)s.rpm' + 'elasticsearch-%(version)s.tar.gz', + 'elasticsearch-%(version)s.zip', + 'elasticsearch-%(version)s.deb', + 'elasticsearch-%(version)s.rpm' ]] verify_java_version('1.8') if url: download_url = url else: - download_url = '%s/%s-%s' % ('http://download.elasticsearch.org/elasticsearch/staging', version, hash) + download_url = 'https://staging.elastic.co/%s-%s/downloads/elasticsearch' % (version, hash) download_and_verify(version, hash, files, download_url, plugins=DEFAULT_PLUGINS + plugins)