diff --git a/dev-tools/scripts/poll-mirrors.pl b/dev-tools/scripts/poll-mirrors.pl deleted file mode 100755 index cdbd3bb68fb..00000000000 --- a/dev-tools/scripts/poll-mirrors.pl +++ /dev/null @@ -1,155 +0,0 @@ -#!/usr/bin/perl -# -# poll-mirrors.pl -# -# This script is designed to poll download sites after posting a release -# and print out notice as each becomes available. The RM can use this -# script to delay the release announcement until the release can be -# downloaded. -# -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -use strict; -use warnings; -use Getopt::Long; -use POSIX qw/strftime/; -use LWP::UserAgent; - -my $rel_path; -my $version; -my $interval = 300; -my $details = 0; - -my $result = GetOptions ("version=s" => \$version, - "details!" => \$details, - "path=s" => \$rel_path, - "interval=i" => \$interval); - -my $usage = "" - . "$0 -v version [ -i interval (seconds; default: 300) ] [ -details ]\n" - . "$0 -p some/explicit/path [ -i interval (seconds; default: 300) ] [ -details ]\n" - ; - -unless ($result) { - print STDERR $usage; - exit(1); -} - -unless (defined($version) xor defined($rel_path)) { - print STDERR "You must specify either -version or -path but not both\n$usage"; - exit(1); -} - -my $label; -my $apache_url_suffix; -my $maven_url; - -if (defined($version)) { - if ($version !~ /^\d+(?:\.\d+)+/) { - print STDERR "You must specify the release version as a number.\n$usage"; - exit(1); - } - $label = $version; - $apache_url_suffix = "lucene/java/$version/changes/Changes.html"; - $maven_url = "http://repo1.maven.org/maven2/org/apache/lucene/lucene-core/$version/lucene-core-$version.pom.asc"; -} else { - # path based - $apache_url_suffix = $label = $rel_path; -} -my $previously_selected = select STDOUT; -$| = 1; # turn off buffering of STDOUT, so status is printed immediately -select $previously_selected; - -my $apache_mirrors_list_url = "http://www.apache.org/mirrors/"; - -my $agent = LWP::UserAgent->new(); -$agent->timeout(2); - -my $maven_available = defined($maven_url) ? 0 : -999; - -my @apache_mirrors = (); - -my $apache_mirrors_list_page = $agent->get($apache_mirrors_list_url)->decoded_content; -if (defined($apache_mirrors_list_page)) { - # - # apache.dattatec.com  @ - # - # http - # 8 hours
- # 5 hours
- # ok - # - while ($apache_mirrors_list_page =~ m~(.*?)~gis) { - my $mirror_entry = $1; - next unless ($mirror_entry =~ m~\s*ok\s*\s*$~i); # skip mirrors with problems - if ($mirror_entry =~ m~~i) { - my $mirror_url = $1; - push @apache_mirrors, "${mirror_url}${apache_url_suffix}"; - } - } -} else { - print STDERR "Error fetching Apache mirrors list $apache_mirrors_list_url"; - exit(1); -} - -my $num_apache_mirrors = $#apache_mirrors; - -my $sleep_interval = 0; -while (1) { - print "\n", strftime('%d-%b-%Y %H:%M:%S', localtime); - print "\nPolling $#apache_mirrors Apache Mirrors"; - print " and Maven Central" unless ($maven_available); - print "...\n"; - - my $start = time(); - $maven_available = (200 == $agent->head($maven_url)->code) - unless ($maven_available); - @apache_mirrors = &check_mirrors; - my $stop = time(); - $sleep_interval = $interval - ($stop - $start); - - my $num_downloadable_apache_mirrors = $num_apache_mirrors - $#apache_mirrors; - print "$label is ", ($maven_available ? "" : "not "), - "downloadable from Maven Central.\n" if defined($maven_url); - printf "$label is downloadable from %d/%d Apache Mirrors (%0.1f%%)\n", - $num_downloadable_apache_mirrors, $num_apache_mirrors, - ($num_downloadable_apache_mirrors*100/$num_apache_mirrors); - - last if ($maven_available && 0 == $#apache_mirrors); - - if ($sleep_interval > 0) { - print "Sleeping for $sleep_interval seconds...\n"; - sleep($sleep_interval) - } -} - -sub check_mirrors { - my @not_yet_downloadable_apache_mirrors; - for my $mirror (@apache_mirrors) { - - ### print "\n$mirror\n"; - if (200 != $agent->head($mirror)->code) { - push @not_yet_downloadable_apache_mirrors, $mirror; - print $details ? "\nFAIL: $mirror\n" : "X"; - } else { - print "."; - } - } - print "\n"; - return @not_yet_downloadable_apache_mirrors; -} diff --git a/dev-tools/scripts/poll-mirrors.py b/dev-tools/scripts/poll-mirrors.py new file mode 100644 index 00000000000..1ff7e544ff8 --- /dev/null +++ b/dev-tools/scripts/poll-mirrors.py @@ -0,0 +1,153 @@ +#!/usr/bin/python +# +# vim: softtabstop=2 shiftwidth=2 expandtab +# +# Python port of poll-mirrors.pl +# +# This script is designed to poll download sites after posting a release +# and print out notice as each becomes available. The RM can use this +# script to delay the release announcement until the release can be +# downloaded. +# +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import argparse +import datetime +import ftplib +import re +import sys +import time + +try: + from urllib.parse import urlparse +except: + from urlparse import urlparse + +try: + import http.client as http +except ImportError: + import httplib as http + +def p(s): + sys.stdout.write(s) + sys.stdout.flush() + +def mirror_contains_file(url): + url = urlparse(url) + + if url.scheme == 'http': + return http_file_exists(url) + elif url.scheme == 'ftp': + return ftp_file_exists(url) + +def http_file_exists(url): + exists = False + + try: + conn = http.HTTPConnection(url.netloc) + conn.request('HEAD', url.path) + response = conn.getresponse() + + exists = response.status == 200 + except: + pass + + return exists + +def ftp_file_exists(url): + listing = [] + try: + conn = ftplib.FTP(url.netloc) + conn.login() + listing = conn.nlst(url.path) + conn.quit() + except Exception as e: + pass + + return len(listing) > 0 + +def check_url_list(lst): + ret = [] + for url in lst: + if mirror_contains_file(url): + p('.') + else: + p('X') + ret.append(url) + + return ret + +parser = argparse.ArgumentParser(description='Checks that all Lucene mirrors contain a copy of a release') +parser.add_argument('-version', '-v', help='Lucene version to check', required=True) +parser.add_argument('-interval', '-i', help='seconds to wait to query again pending mirrors', type=int, default=300) +args = parser.parse_args() + +try: + conn = http.HTTPConnection('www.apache.org') + conn.request('GET', '/mirrors/') + response = conn.getresponse() + html = response.read() +except Exception as e: + p('Unable to fetch the Apache mirrors list!\n') + sys.exit(1) + +apache_path = 'lucene/java/{}/changes/Changes.html'.format(args.version); +maven_url = 'http://repo1.maven.org/maven2/' \ + 'org/apache/lucene/lucene-core/{0}/lucene-core-{0}.pom.asc'.format(args.version) +maven_available = False + +pending_mirrors = [] +for match in re.finditer('(.*?)', str(html), re.MULTILINE | re.IGNORECASE | re.DOTALL): + row = match.group(1) + if not 'ok' in row: + # skip bad mirrors + continue + + match = re.search('', row, re.MULTILINE | re.IGNORECASE) + if match: + pending_mirrors.append(match.group(1) + apache_path) + +total_mirrors = len(pending_mirrors) + +while True: + p('\n' + str(datetime.datetime.now())) + p('\nPolling {} Apache Mirrors'.format(len(pending_mirrors))) + if not maven_available: + p(' and Maven Central') + p('...\n') + + if not maven_available: + maven_available = mirror_contains_file(maven_url) + + start = time.time() + pending_mirrors = check_url_list(pending_mirrors) + stop = time.time() + remaining = args.interval - (stop - start) + + available_mirrors = total_mirrors - len(pending_mirrors) + + p('\n\n{} is{}downloadable from Maven Central\n'.format(args.version, maven_available and ' ' or ' not ')) + p('{} is downloadable from {}/{} Apache Mirrors ({:.2f}%)\n'.format(args.version, available_mirrors, + total_mirrors, + available_mirrors * 100 / total_mirrors)) + if len(pending_mirrors) == 0: + break + + if remaining > 0: + p('Sleeping for {} seconds...\n'.format(remaining)) + time.sleep(remaining) +