diff --git a/dev-tools/scripts/README.md b/dev-tools/scripts/README.md index 0aab4021fa1..a15b9c776ad 100644 --- a/dev-tools/scripts/README.md +++ b/dev-tools/scripts/README.md @@ -179,25 +179,6 @@ and prints a regular expression that will match all of them --no-git Do not run "git" at all --iters N Number of iterations per test suite (default: 5) -### poll-mirrors.py - - usage: poll-mirrors.py [-h] [-version VERSION] [-path PATH] - [-interval INTERVAL] [-details] [-once] - - Periodically checks that all Lucene mirrors contain either a copy of a - release or a specified path - - optional arguments: - -h, --help show this help message and exit - -version VERSION, -v VERSION - Lucene version to check - -path PATH, -p PATH instead of a versioned release, check for - some/explicit/path - -interval INTERVAL, -i INTERVAL - seconds to wait before re-querying mirrors - -details, -d print missing mirror URLs - -once, -o run only once - ### githubPRs.py usage: githubPRs.py [-h] [--json] [--token TOKEN] diff --git a/dev-tools/scripts/poll-mirrors.py b/dev-tools/scripts/poll-mirrors.py deleted file mode 100755 index 342ee5e6ff4..00000000000 --- a/dev-tools/scripts/poll-mirrors.py +++ /dev/null @@ -1,175 +0,0 @@ -#!/usr/bin/env python3 -# -# vim: softtabstop=2 shiftwidth=2 expandtab -# -# Python port of poll-mirrors.pl -# -# This script is designed to poll download sites after posting a release -# and print out notice as each becomes available. The RM can use this -# script to delay the release announcement until the release can be -# downloaded. -# -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import argparse -import datetime -import ftplib -import re -import sys -import time - -from urllib.parse import urlparse -from multiprocessing import Pool -import http.client as http - - -def p(s): - sys.stdout.write(s) - sys.stdout.flush() - - -def mirror_contains_file(url): - url = urlparse(url) - - if url.scheme == 'https': - return https_file_exists(url) - elif url.scheme == 'http': - return http_file_exists(url) - elif url.scheme == 'ftp': - return ftp_file_exists(url) - - -def http_file_exists(url): - exists = False - - try: - conn = http.HTTPConnection(url.netloc) - conn.request('HEAD', url.path) - response = conn.getresponse() - - exists = response.status == 200 - except: - pass - - return exists - -def https_file_exists(url): - exists = False - - try: - conn = http.HTTPSConnection(url.netloc) - conn.request('HEAD', url.path) - response = conn.getresponse() - exists = response.status == 200 - except: - pass - - return exists - -def ftp_file_exists(url): - listing = [] - try: - conn = ftplib.FTP(url.netloc) - conn.login() - listing = conn.nlst(url.path) - conn.quit() - except Exception as e: - pass - - return len(listing) > 0 - - -def check_mirror(url): - if mirror_contains_file(url): - p('.') - return None - else: - p('\nFAIL: ' + url + '\n' if args.details else 'X') - return url - - -desc = 'Periodically checks that all Lucene mirrors contain either a copy of a release or a specified path' -parser = argparse.ArgumentParser(description=desc) -parser.add_argument('-version', '-v', help='Lucene version to check') -parser.add_argument('-path', '-p', help='instead of a versioned release, check for some/explicit/path') -parser.add_argument('-interval', '-i', help='seconds to wait before re-querying mirrors', type=int, default=300) -parser.add_argument('-details', '-d', help='print missing mirror URLs', action='store_true', default=False) -parser.add_argument('-once', '-o', help='run only once', action='store_true', default=False) -args = parser.parse_args() - -if (args.version is None and args.path is None) \ - or (args.version is not None and args.path is not None): - p('You must specify either -version or -path but not both!\n') - sys.exit(1) - -try: - conn = http.HTTPConnection('www.apache.org') - conn.request('GET', '/mirrors/') - response = conn.getresponse() - html = response.read() -except Exception as e: - p('Unable to fetch the Apache mirrors list!\n') - sys.exit(1) - -mirror_path = args.path if args.path is not None else 'lucene/java/{}/changes/Changes.html'.format(args.version) -maven_url = None if args.version is None else 'https://repo1.maven.org/maven2/' \ - 'org/apache/lucene/lucene-core/{0}/lucene-core-{0}.pom.asc'.format(args.version) -maven_available = False - -pending_mirrors = [] -for match in re.finditer('(.*?)', str(html), re.MULTILINE | re.IGNORECASE | re.DOTALL): - row = match.group(1) - if not 'ok' in row: - # skip bad mirrors - continue - - match = re.search('', row, re.MULTILINE | re.IGNORECASE) - if match: - pending_mirrors.append(match.group(1) + mirror_path) - -total_mirrors = len(pending_mirrors) - -label = args.version if args.version is not None else args.path -while True: - p('\n{:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now())) - p('\nPolling {} Apache Mirrors'.format(len(pending_mirrors))) - if maven_url is not None and not maven_available: - p(' and Maven Central') - p('...\n') - - if maven_url is not None and not maven_available: - maven_available = mirror_contains_file(maven_url) - - start = time.time() - with Pool(processes=5) as pool: - pending_mirrors = list(filter(lambda x: x is not None, pool.map(check_mirror, pending_mirrors))) - stop = time.time() - remaining = args.interval - (stop - start) - - available_mirrors = total_mirrors - len(pending_mirrors) - - if maven_url is not None: - p('\n\n{} is{}downloadable from Maven Central'.format(label, ' ' if maven_available else ' not ')) - p('\n{} is downloadable from {}/{} Apache Mirrors ({:.2f}%)\n' - .format(label, available_mirrors, total_mirrors, available_mirrors * 100 / total_mirrors)) - if len(pending_mirrors) == 0 or args.once == True: - break - - if remaining > 0: - p('Sleeping for {:d} seconds...\n'.format(int(remaining + 0.5))) - time.sleep(remaining) - diff --git a/dev-tools/scripts/releaseWizard.py b/dev-tools/scripts/releaseWizard.py index fdcaa5702a9..ab97ba1389f 100755 --- a/dev-tools/scripts/releaseWizard.py +++ b/dev-tools/scripts/releaseWizard.py @@ -1951,6 +1951,25 @@ def prepare_announce_lucene(todo): return True +def check_artifacts_available(todo): + try: + cdnUrl = expand_jinja("https://dlcdn.apache.org/lucene/java/{{ release_version }}/lucene-{{ release_version }}-src.tgz.asc") + load(cdnUrl) + print("Found %s" % cdnUrl) + except Exception as e: + print("Could not fetch %s (%s)" % (cdnUrl, e)) + return False + + try: + mavenUrl = expand_jinja("https://repo1.maven.org/maven2/org/apache/lucene/lucene-core/{{ release_version }}/lucene-core-{{ release_version }}.pom.asc") + load(mavenUrl) + print("Found %s" % mavenUrl) + except Exception as e: + print("Could not fetch %s (%s)" % (mavenUrl, e)) + return False + + return True + def set_java_home(version): os.environ['JAVA_HOME'] = state.get_java_home_for_version(version) os.environ['JAVACMD'] = state.get_java_cmd_for_version(version) diff --git a/dev-tools/scripts/releaseWizard.yaml b/dev-tools/scripts/releaseWizard.yaml index 915d3de3465..4259749d06c 100644 --- a/dev-tools/scripts/releaseWizard.yaml +++ b/dev-tools/scripts/releaseWizard.yaml @@ -838,8 +838,8 @@ groups: - https://www.apache.org/foundation/voting.html - !TodoGroup id: publish - title: Publishing to the ASF Mirrors - description: Once the vote has passed, the release may be published to the ASF Mirrors and to Maven Central. + title: Publishing to the ASF Distribution Directory + description: Once the vote has passed, the release may be published to the ASF Distribution Directory and to Maven Central. todos: - !Todo id: tag_release @@ -868,7 +868,7 @@ groups: commands: !Commands root_folder: '{{ git_checkout_folder }}' confirm_each_command: false - commands_text: This will remove maven artifacts so they do not end up in the mirrors + commands_text: This will remove maven artifacts so they do not end up in the Distribution Directory commands: - !Command cmd: svn rm -m "Delete the lucene maven artifacts" {{ dist_stage_url }}/lucene/maven @@ -928,22 +928,22 @@ groups: . Wait and keep clicking refresh until the "Release" button becomes available . Click on the "Release" button above the repository list, then enter a description when prompted, e.g. "Lucene {{ release_version }}". - Maven central should show the release after a short while, but you need to - wait 24 hours to give the Apache mirrors a chance to copy the new release. + Maven central should show the release after a short while links: - https://wiki.apache.org/lucene-java/PublishMavenArtifacts - https://repository.apache.org/index.html - !Todo - id: check_mirroring - title: Check state of mirroring so far - description: Mark this as complete once a good spread is confirmed - commands: !Commands - root_folder: '{{ git_checkout_folder }}' - commands_text: Run this script to check the number and percentage of mirrors (and Maven Central) that have the release - commands: - - !Command - cmd: python3 -u dev-tools/scripts/poll-mirrors.py -version {{ release_version }} -o - live: true + id: check_distribution_directory + depends: publish_maven + title: Check that artifacts are available + function: check_artifacts_available + description: | + The task will attempt to fetch https://dlcdn.apache.org/lucene/java/{{ release_version }}/lucene-{{ release_version }}-src.tgz.asc + to validate ASF repo, and https://repo1.maven.org/maven2/org/apache/lucene/lucene-core/{{ release_version }}/lucene-core-{{ release_version }}.pom.asc + to validate Maven repo. + + If the check fails, please re-run the task, until it succeeds. + - !TodoGroup id: website title: Update the website @@ -1543,32 +1543,32 @@ groups: links: - https://issues.apache.org/jira/plugins/servlet/project-config/LUCENE/versions - !Todo - id: stop_mirroring - title: Stop mirroring old releases + id: stop_promoting_old + title: Stop promoting old releases description: | - Shortly after new releases are first mirrored, they are automatically copied to the archives. + Shortly after new releases are first published, they are automatically copied to the archives. Only the latest point release from each active branch should be kept under the Lucene PMC svnpubsub area `dist/releases/lucene/`. Older releases can be safely deleted, since they are already backed up in the archives. - Currenlty these versions are in the mirrors: + Currenlty these versions exist in the distribution directory: *{{ mirrored_versions|join(', ') }}* The commands below will remove old versions automatically. If this suggestion is wrong, please do *not* execute the commands automatically, but edit the command and run manually. - Versions to be deleted from the mirrors are: + Versions to be deleted from the distribution directory are: *{{ mirrored_versions_to_delete|join(', ') }}* commands: !Commands root_folder: '{{ git_checkout_folder }}' commands_text: | - Run these commands to delete proposed versions from mirrors. + Run these commands to delete proposed versions from distribution directory. WARNING: Validate that the proposal is correct! commands: - !Command cmd: | - svn rm -m "Stop mirroring old Lucene releases"{% for ver in mirrored_versions_to_delete %} https://dist.apache.org/repos/dist/release/lucene/java/{{ ver }}{% endfor %} + svn rm -m "Stop publishing old Lucene releases"{% for ver in mirrored_versions_to_delete %} https://dist.apache.org/repos/dist/release/lucene/java/{{ ver }}{% endfor %} logfile: svn-rm-lucene.log