mirror of https://github.com/apache/lucene.git
get smoke tester working with shortened URL again (python3 upgrade broke it)
git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1389334 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
parent
84c5a4adde
commit
2c783bd178
|
@ -16,6 +16,7 @@
|
||||||
import os
|
import os
|
||||||
import tarfile
|
import tarfile
|
||||||
import threading
|
import threading
|
||||||
|
import traceback
|
||||||
import subprocess
|
import subprocess
|
||||||
import signal
|
import signal
|
||||||
import shutil
|
import shutil
|
||||||
|
@ -43,7 +44,7 @@ def unshortenURL(url):
|
||||||
h = http.client.HTTPConnection(parsed.netloc)
|
h = http.client.HTTPConnection(parsed.netloc)
|
||||||
h.request('HEAD', parsed.path)
|
h.request('HEAD', parsed.path)
|
||||||
response = h.getresponse()
|
response = h.getresponse()
|
||||||
if response.status/100 == 3 and response.getheader('Location'):
|
if int(response.status/100) == 3 and response.getheader('Location'):
|
||||||
return response.getheader('Location')
|
return response.getheader('Location')
|
||||||
return url
|
return url
|
||||||
|
|
||||||
|
@ -112,7 +113,14 @@ def getHREFs(urlString):
|
||||||
break
|
break
|
||||||
|
|
||||||
links = []
|
links = []
|
||||||
for subUrl, text in reHREF.findall(urllib.request.urlopen(urlString).read().decode('UTF-8')):
|
try:
|
||||||
|
html = urllib.request.urlopen(urlString).read().decode('UTF-8')
|
||||||
|
except:
|
||||||
|
print('\nFAILED to open url %s' % urlString)
|
||||||
|
tracekback.print_exc()
|
||||||
|
raise
|
||||||
|
|
||||||
|
for subUrl, text in reHREF.findall(html):
|
||||||
fullURL = urllib.parse.urljoin(urlString, subUrl)
|
fullURL = urllib.parse.urljoin(urlString, subUrl)
|
||||||
links.append((text, fullURL))
|
links.append((text, fullURL))
|
||||||
return links
|
return links
|
||||||
|
|
Loading…
Reference in New Issue