lucene/dev-tools/scripts/addBackcompatIndexes.py

237 lines
8.2 KiB
Python
Executable File

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# For usage information, see:
#
# http://wiki.apache.org/lucene-java/ReleaseTodo#Generate_Backcompat_Indexes
import os
import sys
sys.path.append(os.path.dirname(__file__))
import scriptutil
import argparse
import urllib.error
import urllib.request
import re
import shutil
def create_and_add_index(source, indextype, index_version, current_version, temp_dir):
if not current_version.is_back_compat_with(index_version):
prefix = 'unsupported'
else:
prefix = {
'cfs': 'index',
'nocfs': 'index',
'sorted': 'sorted',
'moreterms': 'moreterms',
'dvupdates': 'dvupdates',
'emptyIndex': 'empty'
}[indextype]
if indextype in ('cfs', 'nocfs'):
filename = '%s.%s-%s.zip' % (prefix, index_version, indextype)
else:
filename = '%s.%s.zip' % (prefix, index_version)
print(' creating %s...' % filename, end='', flush=True)
module = 'backward-codecs'
index_dir = os.path.join('lucene', module, 'src/test/org/apache/lucene/backward_index')
if os.path.exists(os.path.join(index_dir, filename)):
print('uptodate')
return
test = {
'cfs': 'testCreateCFS',
'nocfs': 'testCreateNoCFS',
'sorted': 'testCreateSortedIndex',
'moreterms': 'testCreateMoreTermsIndex',
'dvupdates': 'testCreateIndexWithDocValuesUpdates',
'emptyIndex': 'testCreateEmptyIndex'
}[indextype]
gradle_args = ' '.join([
'-Ptests.useSecurityManager=false',
'-p lucene/%s' % module,
'test',
'--tests TestGenerateBwcIndices.%s' % test,
'-Dtests.bwcdir=%s' % temp_dir,
'-Dtests.codec=default'
])
base_dir = os.getcwd()
bc_index_file = os.path.join(temp_dir, filename)
if os.path.exists(bc_index_file):
print('alreadyexists')
else:
os.chdir(source)
scriptutil.run('./gradlew %s' % gradle_args)
if not os.path.exists(bc_index_file):
raise Exception("Expected file can't be found: %s" %bc_index_file)
print('done')
print(' adding %s...' % filename, end='', flush=True)
scriptutil.run('cp %s %s' % (bc_index_file, os.path.join(base_dir, index_dir)))
os.chdir(base_dir)
print('done')
def update_backcompat_tests(index_version, current_version):
print(' adding new indexes to backcompat tests...', end='', flush=True)
module = 'lucene/backward-codecs'
filename = None
if not current_version.is_back_compat_with(index_version):
filename = '%s/src/test/org/apache/lucene/backward_index/unsupported_versions.txt' % module
else:
filename = '%s/src/test/org/apache/lucene/backward_index/versions.txt' % module
strip_dash_suffix_re = re.compile(r'-.*')
def find_version(x):
x = x.strip()
x = re.sub(strip_dash_suffix_re, '', x) # remove the -suffix if any
return scriptutil.Version.parse(x)
def edit(buffer, match, line):
v = find_version(line)
changed = False
if v.on_or_after(index_version):
if not index_version.on_or_after(v):
buffer.append(('%s\n') % index_version)
changed = True
buffer.append(line)
return changed
def append(buffer, changed):
if changed:
return changed
if not buffer[len(buffer)-1].endswith('\n'):
buffer.append('\n')
buffer.append(('%s\n') % index_version)
return True
changed = scriptutil.update_file(filename, re.compile(r'.*'), edit, append)
print('done' if changed else 'uptodate')
def check_backcompat_tests():
print(' checking backcompat tests...', end='', flush=True)
scriptutil.run('./gradlew -p lucene/backward-codecs test --tests TestGenerateBwcIndices')
print('ok')
def download_from_cdn(version, remotename, localname):
url = 'http://dlcdn.apache.org/lucene/java/%s/%s' % (version, remotename)
try:
urllib.request.urlretrieve(url, localname)
return True
except urllib.error.URLError as e:
if e.code == 404:
return False
raise e
def download_from_archives(version, remotename, localname):
url = 'http://archive.apache.org/dist/lucene/java/%s/%s' % (version, remotename)
try:
urllib.request.urlretrieve(url, localname)
return True
except urllib.error.URLError as e:
if e.code == 404:
return False
raise e
def download_release(version, temp_dir, force):
print(' downloading %s source release...' % version, end='', flush=True)
source = os.path.join(temp_dir, 'lucene-%s' % version)
if os.path.exists(source):
if force:
shutil.rmtree(source)
else:
print('uptodate')
return source
filename = 'lucene-%s-src.tgz' % version
source_tgz = os.path.join(temp_dir, filename)
if not download_from_cdn(version, filename, source_tgz) and \
not download_from_archives(version, filename, source_tgz):
raise Exception('Could not find version %s in apache CDN or archives' % version)
olddir = os.getcwd()
os.chdir(temp_dir)
scriptutil.run('tar -xvzf %s' % source_tgz)
os.chdir(olddir)
print('done')
return source
def read_config():
parser = argparse.ArgumentParser(formatter_class=argparse.RawDescriptionHelpFormatter,
description='''\
Add backcompat index and test for new version. See:
http://wiki.apache.org/lucene-java/ReleaseTodo#Generate_Backcompat_Indexes
''')
parser.add_argument('--force', action='store_true', default=False,
help='Redownload the version and rebuild, even if it already exists')
parser.add_argument('--no-cleanup', dest='cleanup', action='store_false', default=True,
help='Do not cleanup the built indexes, so that they can be reused ' +
'for adding to another branch')
parser.add_argument('--temp-dir', metavar='DIR', default='/tmp/lucenebwc',
help='Temp directory to build backcompat indexes within')
parser.add_argument('version', type=scriptutil.Version.parse,
help='Version to add, of the form X.Y.Z')
c = parser.parse_args()
return c
def main():
c = read_config()
if not os.path.exists(c.temp_dir):
os.makedirs(c.temp_dir)
print('\nCreating backwards compatibility indexes')
source = download_release(c.version, c.temp_dir, c.force)
current_version = scriptutil.Version.parse(scriptutil.find_current_version())
create_and_add_index(source, 'cfs', c.version, current_version, c.temp_dir)
create_and_add_index(source, 'nocfs', c.version, current_version, c.temp_dir)
should_make_sorted = current_version.is_back_compat_with(c.version) \
and (c.version.major > 6 or (c.version.major == 6 and c.version.minor >= 2))
if should_make_sorted:
create_and_add_index(source, 'sorted', c.version, current_version, c.temp_dir)
if c.version.minor == 0 and c.version.bugfix == 0 and current_version.is_back_compat_with(c.version):
create_and_add_index(source, 'moreterms', c.version, current_version, c.temp_dir)
create_and_add_index(source, 'dvupdates', c.version, current_version, c.temp_dir)
create_and_add_index(source, 'emptyIndex', c.version, current_version, c.temp_dir)
print ('\nMANUAL UPDATE REQUIRED: edit TestGenerateBwcIndices to enable moreterms, dvupdates, and empty index testing')
print('\nAdding backwards compatibility tests')
update_backcompat_tests(c.version, current_version)
print('\nTesting changes')
check_backcompat_tests()
if c.cleanup:
print('\nCleaning up')
print(' deleting %s...' % c.temp_dir, end='', flush=True)
shutil.rmtree(c.temp_dir)
print('done')
print()
if __name__ == '__main__':
try:
main()
except KeyboardInterrupt:
print('\nRecieved Ctrl-C, exiting early')