mirror of https://github.com/apache/lucene.git
LUCENE-7425: change from Perl to Python for our script to polling mirrors during the release process
This commit is contained in:
Mike McCandless
parent
8683da80ed
commit
d44e7315b4
|
|
@ -1,155 +0,0 @@
|
|||
#!/usr/bin/perl
|
||||
#
|
||||
# poll-mirrors.pl
|
||||
#
|
||||
# This script is designed to poll download sites after posting a release
|
||||
# and print out notice as each becomes available. The RM can use this
|
||||
# script to delay the release announcement until the release can be
|
||||
# downloaded.
|
||||
#
|
||||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
use strict;
|
||||
use warnings;
|
||||
use Getopt::Long;
|
||||
use POSIX qw/strftime/;
|
||||
use LWP::UserAgent;
|
||||
|
||||
my $rel_path;
|
||||
my $version;
|
||||
my $interval = 300;
|
||||
my $details = 0;
|
||||
|
||||
my $result = GetOptions ("version=s" => \$version,
|
||||
"details!" => \$details,
|
||||
"path=s" => \$rel_path,
|
||||
"interval=i" => \$interval);
|
||||
|
||||
my $usage = ""
|
||||
. "$0 -v version [ -i interval (seconds; default: 300) ] [ -details ]\n"
|
||||
. "$0 -p some/explicit/path [ -i interval (seconds; default: 300) ] [ -details ]\n"
|
||||
;
|
||||
|
||||
unless ($result) {
|
||||
print STDERR $usage;
|
||||
exit(1);
|
||||
}
|
||||
|
||||
unless (defined($version) xor defined($rel_path)) {
|
||||
print STDERR "You must specify either -version or -path but not both\n$usage";
|
||||
exit(1);
|
||||
}
|
||||
|
||||
my $label;
|
||||
my $apache_url_suffix;
|
||||
my $maven_url;
|
||||
|
||||
if (defined($version)) {
|
||||
if ($version !~ /^\d+(?:\.\d+)+/) {
|
||||
print STDERR "You must specify the release version as a number.\n$usage";
|
||||
exit(1);
|
||||
}
|
||||
$label = $version;
|
||||
$apache_url_suffix = "lucene/java/$version/changes/Changes.html";
|
||||
$maven_url = "http://repo1.maven.org/maven2/org/apache/lucene/lucene-core/$version/lucene-core-$version.pom.asc";
|
||||
} else {
|
||||
# path based
|
||||
$apache_url_suffix = $label = $rel_path;
|
||||
}
|
||||
my $previously_selected = select STDOUT;
|
||||
$| = 1; # turn off buffering of STDOUT, so status is printed immediately
|
||||
select $previously_selected;
|
||||
|
||||
my $apache_mirrors_list_url = "http://www.apache.org/mirrors/";
|
||||
|
||||
my $agent = LWP::UserAgent->new();
|
||||
$agent->timeout(2);
|
||||
|
||||
my $maven_available = defined($maven_url) ? 0 : -999;
|
||||
|
||||
my @apache_mirrors = ();
|
||||
|
||||
my $apache_mirrors_list_page = $agent->get($apache_mirrors_list_url)->decoded_content;
|
||||
if (defined($apache_mirrors_list_page)) {
|
||||
#<TR>
|
||||
# <TD ALIGN=RIGHT><A HREF="http://apache.dattatec.com/">apache.dattatec.com</A> <A HREF="http://apache.dattatec.com/">@</A></TD>
|
||||
#
|
||||
# <TD>http</TD>
|
||||
# <TD ALIGN=RIGHT>8 hours<BR><IMG BORDER=1 SRC="icons/mms14.gif" ALT=""></TD>
|
||||
# <TD ALIGN=RIGHT>5 hours<BR><IMG BORDER=1 SRC="icons/mms14.gif" ALT=""></TD>
|
||||
# <TD>ok</TD>
|
||||
#</TR>
|
||||
while ($apache_mirrors_list_page =~ m~<TR>(.*?)</TR>~gis) {
|
||||
my $mirror_entry = $1;
|
||||
next unless ($mirror_entry =~ m~<TD>\s*ok\s*</TD>\s*$~i); # skip mirrors with problems
|
||||
if ($mirror_entry =~ m~<A\s+HREF\s*=\s*"([^"]+)"\s*>~i) {
|
||||
my $mirror_url = $1;
|
||||
push @apache_mirrors, "${mirror_url}${apache_url_suffix}";
|
||||
}
|
||||
}
|
||||
} else {
|
||||
print STDERR "Error fetching Apache mirrors list $apache_mirrors_list_url";
|
||||
exit(1);
|
||||
}
|
||||
|
||||
my $num_apache_mirrors = $#apache_mirrors;
|
||||
|
||||
my $sleep_interval = 0;
|
||||
while (1) {
|
||||
print "\n", strftime('%d-%b-%Y %H:%M:%S', localtime);
|
||||
print "\nPolling $#apache_mirrors Apache Mirrors";
|
||||
print " and Maven Central" unless ($maven_available);
|
||||
print "...\n";
|
||||
|
||||
my $start = time();
|
||||
$maven_available = (200 == $agent->head($maven_url)->code)
|
||||
unless ($maven_available);
|
||||
@apache_mirrors = &check_mirrors;
|
||||
my $stop = time();
|
||||
$sleep_interval = $interval - ($stop - $start);
|
||||
|
||||
my $num_downloadable_apache_mirrors = $num_apache_mirrors - $#apache_mirrors;
|
||||
print "$label is ", ($maven_available ? "" : "not "),
|
||||
"downloadable from Maven Central.\n" if defined($maven_url);
|
||||
printf "$label is downloadable from %d/%d Apache Mirrors (%0.1f%%)\n",
|
||||
$num_downloadable_apache_mirrors, $num_apache_mirrors,
|
||||
($num_downloadable_apache_mirrors*100/$num_apache_mirrors);
|
||||
|
||||
last if ($maven_available && 0 == $#apache_mirrors);
|
||||
|
||||
if ($sleep_interval > 0) {
|
||||
print "Sleeping for $sleep_interval seconds...\n";
|
||||
sleep($sleep_interval)
|
||||
}
|
||||
}
|
||||
|
||||
sub check_mirrors {
|
||||
my @not_yet_downloadable_apache_mirrors;
|
||||
for my $mirror (@apache_mirrors) {
|
||||
|
||||
### print "\n$mirror\n";
|
||||
if (200 != $agent->head($mirror)->code) {
|
||||
push @not_yet_downloadable_apache_mirrors, $mirror;
|
||||
print $details ? "\nFAIL: $mirror\n" : "X";
|
||||
} else {
|
||||
print ".";
|
||||
}
|
||||
}
|
||||
print "\n";
|
||||
return @not_yet_downloadable_apache_mirrors;
|
||||
}
|
||||
|
|
@ -0,0 +1,153 @@
|
|||
#!/usr/bin/python
|
||||
#
|
||||
# vim: softtabstop=2 shiftwidth=2 expandtab
|
||||
#
|
||||
# Python port of poll-mirrors.pl
|
||||
#
|
||||
# This script is designed to poll download sites after posting a release
|
||||
# and print out notice as each becomes available. The RM can use this
|
||||
# script to delay the release announcement until the release can be
|
||||
# downloaded.
|
||||
#
|
||||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
import argparse
|
||||
import datetime
|
||||
import ftplib
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
|
||||
try:
|
||||
from urllib.parse import urlparse
|
||||
except:
|
||||
from urlparse import urlparse
|
||||
|
||||
try:
|
||||
import http.client as http
|
||||
except ImportError:
|
||||
import httplib as http
|
||||
|
||||
def p(s):
|
||||
sys.stdout.write(s)
|
||||
sys.stdout.flush()
|
||||
|
||||
def mirror_contains_file(url):
|
||||
url = urlparse(url)
|
||||
|
||||
if url.scheme == 'http':
|
||||
return http_file_exists(url)
|
||||
elif url.scheme == 'ftp':
|
||||
return ftp_file_exists(url)
|
||||
|
||||
def http_file_exists(url):
|
||||
exists = False
|
||||
|
||||
try:
|
||||
conn = http.HTTPConnection(url.netloc)
|
||||
conn.request('HEAD', url.path)
|
||||
response = conn.getresponse()
|
||||
|
||||
exists = response.status == 200
|
||||
except:
|
||||
pass
|
||||
|
||||
return exists
|
||||
|
||||
def ftp_file_exists(url):
|
||||
listing = []
|
||||
try:
|
||||
conn = ftplib.FTP(url.netloc)
|
||||
conn.login()
|
||||
listing = conn.nlst(url.path)
|
||||
conn.quit()
|
||||
except Exception as e:
|
||||
pass
|
||||
|
||||
return len(listing) > 0
|
||||
|
||||
def check_url_list(lst):
|
||||
ret = []
|
||||
for url in lst:
|
||||
if mirror_contains_file(url):
|
||||
p('.')
|
||||
else:
|
||||
p('X')
|
||||
ret.append(url)
|
||||
|
||||
return ret
|
||||
|
||||
parser = argparse.ArgumentParser(description='Checks that all Lucene mirrors contain a copy of a release')
|
||||
parser.add_argument('-version', '-v', help='Lucene version to check', required=True)
|
||||
parser.add_argument('-interval', '-i', help='seconds to wait to query again pending mirrors', type=int, default=300)
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
conn = http.HTTPConnection('www.apache.org')
|
||||
conn.request('GET', '/mirrors/')
|
||||
response = conn.getresponse()
|
||||
html = response.read()
|
||||
except Exception as e:
|
||||
p('Unable to fetch the Apache mirrors list!\n')
|
||||
sys.exit(1)
|
||||
|
||||
apache_path = 'lucene/java/{}/changes/Changes.html'.format(args.version);
|
||||
maven_url = 'http://repo1.maven.org/maven2/' \
|
||||
'org/apache/lucene/lucene-core/{0}/lucene-core-{0}.pom.asc'.format(args.version)
|
||||
maven_available = False
|
||||
|
||||
pending_mirrors = []
|
||||
for match in re.finditer('<TR>(.*?)</TR>', str(html), re.MULTILINE | re.IGNORECASE | re.DOTALL):
|
||||
row = match.group(1)
|
||||
if not '<TD>ok</TD>' in row:
|
||||
# skip bad mirrors
|
||||
continue
|
||||
|
||||
match = re.search('<A\s+HREF\s*=\s*"([^"]+)"\s*>', row, re.MULTILINE | re.IGNORECASE)
|
||||
if match:
|
||||
pending_mirrors.append(match.group(1) + apache_path)
|
||||
|
||||
total_mirrors = len(pending_mirrors)
|
||||
|
||||
while True:
|
||||
p('\n' + str(datetime.datetime.now()))
|
||||
p('\nPolling {} Apache Mirrors'.format(len(pending_mirrors)))
|
||||
if not maven_available:
|
||||
p(' and Maven Central')
|
||||
p('...\n')
|
||||
|
||||
if not maven_available:
|
||||
maven_available = mirror_contains_file(maven_url)
|
||||
|
||||
start = time.time()
|
||||
pending_mirrors = check_url_list(pending_mirrors)
|
||||
stop = time.time()
|
||||
remaining = args.interval - (stop - start)
|
||||
|
||||
available_mirrors = total_mirrors - len(pending_mirrors)
|
||||
|
||||
p('\n\n{} is{}downloadable from Maven Central\n'.format(args.version, maven_available and ' ' or ' not '))
|
||||
p('{} is downloadable from {}/{} Apache Mirrors ({:.2f}%)\n'.format(args.version, available_mirrors,
|
||||
total_mirrors,
|
||||
available_mirrors * 100 / total_mirrors))
|
||||
if len(pending_mirrors) == 0:
|
||||
break
|
||||
|
||||
if remaining > 0:
|
||||
p('Sleeping for {} seconds...\n'.format(remaining))
|
||||
time.sleep(remaining)
|
||||
|
||||
Loading…
Reference in New Issue