mirror of https://github.com/apache/lucene.git
LUCENE-10179 No longer check for release status on mirrors (#384)
This commit is contained in:
Jan Høydahl
GitHub
parent
560f71b47d
commit
f38c401283
|
|
@ -179,25 +179,6 @@ and prints a regular expression that will match all of them
|
|||
--no-git Do not run "git" at all
|
||||
--iters N Number of iterations per test suite (default: 5)
|
||||
|
||||
### poll-mirrors.py
|
||||
|
||||
usage: poll-mirrors.py [-h] [-version VERSION] [-path PATH]
|
||||
[-interval INTERVAL] [-details] [-once]
|
||||
|
||||
Periodically checks that all Lucene mirrors contain either a copy of a
|
||||
release or a specified path
|
||||
|
||||
optional arguments:
|
||||
-h, --help show this help message and exit
|
||||
-version VERSION, -v VERSION
|
||||
Lucene version to check
|
||||
-path PATH, -p PATH instead of a versioned release, check for
|
||||
some/explicit/path
|
||||
-interval INTERVAL, -i INTERVAL
|
||||
seconds to wait before re-querying mirrors
|
||||
-details, -d print missing mirror URLs
|
||||
-once, -o run only once
|
||||
|
||||
### githubPRs.py
|
||||
|
||||
usage: githubPRs.py [-h] [--json] [--token TOKEN]
|
||||
|
|
|
|||
|
|
@ -1,175 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
#
|
||||
# vim: softtabstop=2 shiftwidth=2 expandtab
|
||||
#
|
||||
# Python port of poll-mirrors.pl
|
||||
#
|
||||
# This script is designed to poll download sites after posting a release
|
||||
# and print out notice as each becomes available. The RM can use this
|
||||
# script to delay the release announcement until the release can be
|
||||
# downloaded.
|
||||
#
|
||||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
import argparse
|
||||
import datetime
|
||||
import ftplib
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
|
||||
from urllib.parse import urlparse
|
||||
from multiprocessing import Pool
|
||||
import http.client as http
|
||||
|
||||
|
||||
def p(s):
|
||||
sys.stdout.write(s)
|
||||
sys.stdout.flush()
|
||||
|
||||
|
||||
def mirror_contains_file(url):
|
||||
url = urlparse(url)
|
||||
|
||||
if url.scheme == 'https':
|
||||
return https_file_exists(url)
|
||||
elif url.scheme == 'http':
|
||||
return http_file_exists(url)
|
||||
elif url.scheme == 'ftp':
|
||||
return ftp_file_exists(url)
|
||||
|
||||
|
||||
def http_file_exists(url):
|
||||
exists = False
|
||||
|
||||
try:
|
||||
conn = http.HTTPConnection(url.netloc)
|
||||
conn.request('HEAD', url.path)
|
||||
response = conn.getresponse()
|
||||
|
||||
exists = response.status == 200
|
||||
except:
|
||||
pass
|
||||
|
||||
return exists
|
||||
|
||||
def https_file_exists(url):
|
||||
exists = False
|
||||
|
||||
try:
|
||||
conn = http.HTTPSConnection(url.netloc)
|
||||
conn.request('HEAD', url.path)
|
||||
response = conn.getresponse()
|
||||
exists = response.status == 200
|
||||
except:
|
||||
pass
|
||||
|
||||
return exists
|
||||
|
||||
def ftp_file_exists(url):
|
||||
listing = []
|
||||
try:
|
||||
conn = ftplib.FTP(url.netloc)
|
||||
conn.login()
|
||||
listing = conn.nlst(url.path)
|
||||
conn.quit()
|
||||
except Exception as e:
|
||||
pass
|
||||
|
||||
return len(listing) > 0
|
||||
|
||||
|
||||
def check_mirror(url):
|
||||
if mirror_contains_file(url):
|
||||
p('.')
|
||||
return None
|
||||
else:
|
||||
p('\nFAIL: ' + url + '\n' if args.details else 'X')
|
||||
return url
|
||||
|
||||
|
||||
desc = 'Periodically checks that all Lucene mirrors contain either a copy of a release or a specified path'
|
||||
parser = argparse.ArgumentParser(description=desc)
|
||||
parser.add_argument('-version', '-v', help='Lucene version to check')
|
||||
parser.add_argument('-path', '-p', help='instead of a versioned release, check for some/explicit/path')
|
||||
parser.add_argument('-interval', '-i', help='seconds to wait before re-querying mirrors', type=int, default=300)
|
||||
parser.add_argument('-details', '-d', help='print missing mirror URLs', action='store_true', default=False)
|
||||
parser.add_argument('-once', '-o', help='run only once', action='store_true', default=False)
|
||||
args = parser.parse_args()
|
||||
|
||||
if (args.version is None and args.path is None) \
|
||||
or (args.version is not None and args.path is not None):
|
||||
p('You must specify either -version or -path but not both!\n')
|
||||
sys.exit(1)
|
||||
|
||||
try:
|
||||
conn = http.HTTPConnection('www.apache.org')
|
||||
conn.request('GET', '/mirrors/')
|
||||
response = conn.getresponse()
|
||||
html = response.read()
|
||||
except Exception as e:
|
||||
p('Unable to fetch the Apache mirrors list!\n')
|
||||
sys.exit(1)
|
||||
|
||||
mirror_path = args.path if args.path is not None else 'lucene/java/{}/changes/Changes.html'.format(args.version)
|
||||
maven_url = None if args.version is None else 'https://repo1.maven.org/maven2/' \
|
||||
'org/apache/lucene/lucene-core/{0}/lucene-core-{0}.pom.asc'.format(args.version)
|
||||
maven_available = False
|
||||
|
||||
pending_mirrors = []
|
||||
for match in re.finditer('<TR>(.*?)</TR>', str(html), re.MULTILINE | re.IGNORECASE | re.DOTALL):
|
||||
row = match.group(1)
|
||||
if not '<TD>ok</TD>' in row:
|
||||
# skip bad mirrors
|
||||
continue
|
||||
|
||||
match = re.search('<A\s+HREF\s*=\s*"([^"]+)"\s*>', row, re.MULTILINE | re.IGNORECASE)
|
||||
if match:
|
||||
pending_mirrors.append(match.group(1) + mirror_path)
|
||||
|
||||
total_mirrors = len(pending_mirrors)
|
||||
|
||||
label = args.version if args.version is not None else args.path
|
||||
while True:
|
||||
p('\n{:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now()))
|
||||
p('\nPolling {} Apache Mirrors'.format(len(pending_mirrors)))
|
||||
if maven_url is not None and not maven_available:
|
||||
p(' and Maven Central')
|
||||
p('...\n')
|
||||
|
||||
if maven_url is not None and not maven_available:
|
||||
maven_available = mirror_contains_file(maven_url)
|
||||
|
||||
start = time.time()
|
||||
with Pool(processes=5) as pool:
|
||||
pending_mirrors = list(filter(lambda x: x is not None, pool.map(check_mirror, pending_mirrors)))
|
||||
stop = time.time()
|
||||
remaining = args.interval - (stop - start)
|
||||
|
||||
available_mirrors = total_mirrors - len(pending_mirrors)
|
||||
|
||||
if maven_url is not None:
|
||||
p('\n\n{} is{}downloadable from Maven Central'.format(label, ' ' if maven_available else ' not '))
|
||||
p('\n{} is downloadable from {}/{} Apache Mirrors ({:.2f}%)\n'
|
||||
.format(label, available_mirrors, total_mirrors, available_mirrors * 100 / total_mirrors))
|
||||
if len(pending_mirrors) == 0 or args.once == True:
|
||||
break
|
||||
|
||||
if remaining > 0:
|
||||
p('Sleeping for {:d} seconds...\n'.format(int(remaining + 0.5)))
|
||||
time.sleep(remaining)
|
||||
|
||||
|
|
@ -1951,6 +1951,25 @@ def prepare_announce_lucene(todo):
|
|||
return True
|
||||
|
||||
|
||||
def check_artifacts_available(todo):
|
||||
try:
|
||||
cdnUrl = expand_jinja("https://dlcdn.apache.org/lucene/java/{{ release_version }}/lucene-{{ release_version }}-src.tgz.asc")
|
||||
load(cdnUrl)
|
||||
print("Found %s" % cdnUrl)
|
||||
except Exception as e:
|
||||
print("Could not fetch %s (%s)" % (cdnUrl, e))
|
||||
return False
|
||||
|
||||
try:
|
||||
mavenUrl = expand_jinja("https://repo1.maven.org/maven2/org/apache/lucene/lucene-core/{{ release_version }}/lucene-core-{{ release_version }}.pom.asc")
|
||||
load(mavenUrl)
|
||||
print("Found %s" % mavenUrl)
|
||||
except Exception as e:
|
||||
print("Could not fetch %s (%s)" % (mavenUrl, e))
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
def set_java_home(version):
|
||||
os.environ['JAVA_HOME'] = state.get_java_home_for_version(version)
|
||||
os.environ['JAVACMD'] = state.get_java_cmd_for_version(version)
|
||||
|
|
|
|||
|
|
@ -838,8 +838,8 @@ groups:
|
|||
- https://www.apache.org/foundation/voting.html
|
||||
- !TodoGroup
|
||||
id: publish
|
||||
title: Publishing to the ASF Mirrors
|
||||
description: Once the vote has passed, the release may be published to the ASF Mirrors and to Maven Central.
|
||||
title: Publishing to the ASF Distribution Directory
|
||||
description: Once the vote has passed, the release may be published to the ASF Distribution Directory and to Maven Central.
|
||||
todos:
|
||||
- !Todo
|
||||
id: tag_release
|
||||
|
|
@ -868,7 +868,7 @@ groups:
|
|||
commands: !Commands
|
||||
root_folder: '{{ git_checkout_folder }}'
|
||||
confirm_each_command: false
|
||||
commands_text: This will remove maven artifacts so they do not end up in the mirrors
|
||||
commands_text: This will remove maven artifacts so they do not end up in the Distribution Directory
|
||||
commands:
|
||||
- !Command
|
||||
cmd: svn rm -m "Delete the lucene maven artifacts" {{ dist_stage_url }}/lucene/maven
|
||||
|
|
@ -928,22 +928,22 @@ groups:
|
|||
. Wait and keep clicking refresh until the "Release" button becomes available
|
||||
. Click on the "Release" button above the repository list, then enter a description when prompted, e.g. "Lucene {{ release_version }}".
|
||||
|
||||
Maven central should show the release after a short while, but you need to
|
||||
wait 24 hours to give the Apache mirrors a chance to copy the new release.
|
||||
Maven central should show the release after a short while
|
||||
links:
|
||||
- https://wiki.apache.org/lucene-java/PublishMavenArtifacts
|
||||
- https://repository.apache.org/index.html
|
||||
- !Todo
|
||||
id: check_mirroring
|
||||
title: Check state of mirroring so far
|
||||
description: Mark this as complete once a good spread is confirmed
|
||||
commands: !Commands
|
||||
root_folder: '{{ git_checkout_folder }}'
|
||||
commands_text: Run this script to check the number and percentage of mirrors (and Maven Central) that have the release
|
||||
commands:
|
||||
- !Command
|
||||
cmd: python3 -u dev-tools/scripts/poll-mirrors.py -version {{ release_version }} -o
|
||||
live: true
|
||||
id: check_distribution_directory
|
||||
depends: publish_maven
|
||||
title: Check that artifacts are available
|
||||
function: check_artifacts_available
|
||||
description: |
|
||||
The task will attempt to fetch https://dlcdn.apache.org/lucene/java/{{ release_version }}/lucene-{{ release_version }}-src.tgz.asc
|
||||
to validate ASF repo, and https://repo1.maven.org/maven2/org/apache/lucene/lucene-core/{{ release_version }}/lucene-core-{{ release_version }}.pom.asc
|
||||
to validate Maven repo.
|
||||
|
||||
If the check fails, please re-run the task, until it succeeds.
|
||||
|
||||
- !TodoGroup
|
||||
id: website
|
||||
title: Update the website
|
||||
|
|
@ -1543,32 +1543,32 @@ groups:
|
|||
links:
|
||||
- https://issues.apache.org/jira/plugins/servlet/project-config/LUCENE/versions
|
||||
- !Todo
|
||||
id: stop_mirroring
|
||||
title: Stop mirroring old releases
|
||||
id: stop_promoting_old
|
||||
title: Stop promoting old releases
|
||||
description: |
|
||||
Shortly after new releases are first mirrored, they are automatically copied to the archives.
|
||||
Shortly after new releases are first published, they are automatically copied to the archives.
|
||||
Only the latest point release from each active branch should be kept under the Lucene PMC
|
||||
svnpubsub area `dist/releases/lucene/`. Older releases can be
|
||||
safely deleted, since they are already backed up in the archives.
|
||||
|
||||
Currenlty these versions are in the mirrors:
|
||||
Currenlty these versions exist in the distribution directory:
|
||||
|
||||
*{{ mirrored_versions|join(', ') }}*
|
||||
|
||||
The commands below will remove old versions automatically. If this suggestion is wrong,
|
||||
please do *not* execute the commands automatically, but edit the command and run manually.
|
||||
Versions to be deleted from the mirrors are:
|
||||
Versions to be deleted from the distribution directory are:
|
||||
|
||||
*{{ mirrored_versions_to_delete|join(', ') }}*
|
||||
|
||||
commands: !Commands
|
||||
root_folder: '{{ git_checkout_folder }}'
|
||||
commands_text: |
|
||||
Run these commands to delete proposed versions from mirrors.
|
||||
Run these commands to delete proposed versions from distribution directory.
|
||||
|
||||
WARNING: Validate that the proposal is correct!
|
||||
commands:
|
||||
- !Command
|
||||
cmd: |
|
||||
svn rm -m "Stop mirroring old Lucene releases"{% for ver in mirrored_versions_to_delete %} https://dist.apache.org/repos/dist/release/lucene/java/{{ ver }}{% endfor %}
|
||||
svn rm -m "Stop publishing old Lucene releases"{% for ver in mirrored_versions_to_delete %} https://dist.apache.org/repos/dist/release/lucene/java/{{ ver }}{% endfor %}
|
||||
logfile: svn-rm-lucene.log
|
||||
|
|
|
|||
Loading…
Reference in New Issue