2019-06-20 08:45:17 -04:00
|
|
|
#!/usr/bin/env python3
|
|
|
|
# -*- coding: utf-8 -*-
|
2014-09-24 16:48:57 -04:00
|
|
|
# Licensed to the Apache Software Foundation (ASF) under one or more
|
|
|
|
# contributor license agreements. See the NOTICE file distributed with
|
|
|
|
# this work for additional information regarding copyright ownership.
|
|
|
|
# The ASF licenses this file to You under the Apache License, Version 2.0
|
|
|
|
# (the "License"); you may not use this file except in compliance with
|
|
|
|
# the License. You may obtain a copy of the License at
|
|
|
|
#
|
|
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
#
|
|
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
# See the License for the specific language governing permissions and
|
|
|
|
# limitations under the License.
|
|
|
|
|
2015-04-15 17:36:00 -04:00
|
|
|
|
|
|
|
# For usage information, see:
|
|
|
|
#
|
|
|
|
# http://wiki.apache.org/lucene-java/ReleaseTodo#Generate_Backcompat_Indexes
|
|
|
|
|
|
|
|
|
2014-09-24 16:48:57 -04:00
|
|
|
import os
|
|
|
|
import sys
|
|
|
|
sys.path.append(os.path.dirname(__file__))
|
|
|
|
import scriptutil
|
|
|
|
|
|
|
|
import argparse
|
|
|
|
import urllib.error
|
|
|
|
import urllib.request
|
|
|
|
import re
|
|
|
|
import shutil
|
|
|
|
|
2016-06-25 14:26:42 -04:00
|
|
|
def create_and_add_index(source, indextype, index_version, current_version, temp_dir):
|
2017-09-30 20:44:01 -04:00
|
|
|
if not current_version.is_back_compat_with(index_version):
|
|
|
|
prefix = 'unsupported'
|
|
|
|
else:
|
|
|
|
prefix = {
|
|
|
|
'cfs': 'index',
|
|
|
|
'nocfs': 'index',
|
|
|
|
'sorted': 'sorted',
|
|
|
|
'moreterms': 'moreterms',
|
|
|
|
'dvupdates': 'dvupdates',
|
|
|
|
'emptyIndex': 'empty'
|
|
|
|
}[indextype]
|
2014-09-24 16:48:57 -04:00
|
|
|
if indextype in ('cfs', 'nocfs'):
|
|
|
|
dirname = 'index.%s' % indextype
|
2017-09-30 20:44:01 -04:00
|
|
|
filename = '%s.%s-%s.zip' % (prefix, index_version, indextype)
|
2014-09-24 16:48:57 -04:00
|
|
|
else:
|
|
|
|
dirname = indextype
|
2017-09-30 20:44:01 -04:00
|
|
|
filename = '%s.%s.zip' % (prefix, index_version)
|
|
|
|
|
2014-09-24 16:48:57 -04:00
|
|
|
print(' creating %s...' % filename, end='', flush=True)
|
2015-02-23 01:50:47 -05:00
|
|
|
module = 'backward-codecs'
|
|
|
|
index_dir = os.path.join('lucene', module, 'src/test/org/apache/lucene/index')
|
2014-09-24 16:48:57 -04:00
|
|
|
test_file = os.path.join(index_dir, filename)
|
|
|
|
if os.path.exists(os.path.join(index_dir, filename)):
|
|
|
|
print('uptodate')
|
|
|
|
return
|
|
|
|
|
|
|
|
test = {
|
|
|
|
'cfs': 'testCreateCFS',
|
2017-09-30 20:44:01 -04:00
|
|
|
'nocfs': 'testCreateNoCFS',
|
|
|
|
'sorted': 'testCreateSortedIndex',
|
|
|
|
'moreterms': 'testCreateMoreTermsIndex',
|
|
|
|
'dvupdates': 'testCreateIndexWithDocValuesUpdates',
|
|
|
|
'emptyIndex': 'testCreateEmptyIndex'
|
2014-09-24 16:48:57 -04:00
|
|
|
}[indextype]
|
|
|
|
ant_args = ' '.join([
|
|
|
|
'-Dtests.bwcdir=%s' % temp_dir,
|
|
|
|
'-Dtests.codec=default',
|
|
|
|
'-Dtests.useSecurityManager=false',
|
|
|
|
'-Dtestcase=TestBackwardsCompatibility',
|
|
|
|
'-Dtestmethod=%s' % test
|
|
|
|
])
|
|
|
|
base_dir = os.getcwd()
|
|
|
|
bc_index_dir = os.path.join(temp_dir, dirname)
|
|
|
|
bc_index_file = os.path.join(bc_index_dir, filename)
|
|
|
|
|
|
|
|
if os.path.exists(bc_index_file):
|
|
|
|
print('alreadyexists')
|
|
|
|
else:
|
|
|
|
if os.path.exists(bc_index_dir):
|
|
|
|
shutil.rmtree(bc_index_dir)
|
|
|
|
os.chdir(os.path.join(source, module))
|
|
|
|
scriptutil.run('ant test %s' % ant_args)
|
|
|
|
os.chdir(bc_index_dir)
|
|
|
|
scriptutil.run('zip %s *' % filename)
|
|
|
|
print('done')
|
|
|
|
|
|
|
|
print(' adding %s...' % filename, end='', flush=True)
|
|
|
|
scriptutil.run('cp %s %s' % (bc_index_file, os.path.join(base_dir, index_dir)))
|
|
|
|
os.chdir(base_dir)
|
|
|
|
output = scriptutil.run('svn status %s' % test_file)
|
|
|
|
if not output.strip():
|
|
|
|
# make sure to only add if the file isn't already in svn (we might be regenerating)
|
|
|
|
scriptutil.run('svn add %s' % test_file)
|
|
|
|
os.chdir(base_dir)
|
|
|
|
scriptutil.run('rm -rf %s' % bc_index_dir)
|
|
|
|
print('done')
|
|
|
|
|
2016-06-25 14:26:42 -04:00
|
|
|
def update_backcompat_tests(types, index_version, current_version):
|
2017-09-30 20:44:01 -04:00
|
|
|
print(' adding new indexes %s to backcompat tests...' % types, end='', flush=True)
|
2014-09-24 16:48:57 -04:00
|
|
|
module = 'lucene/backward-codecs'
|
|
|
|
filename = '%s/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java' % module
|
2017-09-30 20:44:01 -04:00
|
|
|
if not current_version.is_back_compat_with(index_version):
|
2017-10-24 12:27:55 -04:00
|
|
|
matcher = re.compile(r'final String\[\] unsupportedNames = {|};')
|
2017-09-30 20:44:01 -04:00
|
|
|
elif 'sorted' in types:
|
|
|
|
matcher = re.compile(r'final static String\[\] oldSortedNames = {|};')
|
|
|
|
else:
|
|
|
|
matcher = re.compile(r'final static String\[\] oldNames = {|};')
|
|
|
|
|
|
|
|
strip_dash_suffix_re = re.compile(r'-.*')
|
2014-09-24 16:48:57 -04:00
|
|
|
|
|
|
|
def find_version(x):
|
|
|
|
x = x.strip()
|
2017-09-30 20:44:01 -04:00
|
|
|
x = re.sub(strip_dash_suffix_re, '', x) # remove the -suffix if any
|
|
|
|
return scriptutil.Version.parse(x)
|
2014-09-24 16:48:57 -04:00
|
|
|
|
|
|
|
class Edit(object):
|
|
|
|
start = None
|
|
|
|
def __call__(self, buffer, match, line):
|
|
|
|
if self.start:
|
2016-06-25 14:26:42 -04:00
|
|
|
# find where this version should exist
|
2017-09-30 23:20:51 -04:00
|
|
|
i = len(buffer) - 1
|
|
|
|
previous_version_exists = not ('};' in line and buffer[-1].strip().endswith("{"))
|
2017-09-30 20:44:01 -04:00
|
|
|
if previous_version_exists: # Only look if there is a version here
|
2014-09-24 16:48:57 -04:00
|
|
|
v = find_version(buffer[i])
|
2017-09-30 20:44:01 -04:00
|
|
|
while i >= self.start and v.on_or_after(index_version):
|
|
|
|
i -= 1
|
|
|
|
v = find_version(buffer[i])
|
2014-09-24 16:48:57 -04:00
|
|
|
i += 1 # readjust since we skipped past by 1
|
|
|
|
|
|
|
|
# unfortunately python doesn't have a range remove from list...
|
|
|
|
# here we want to remove any previous references to the version we are adding
|
2016-06-25 14:26:42 -04:00
|
|
|
while i < len(buffer) and index_version.on_or_after(find_version(buffer[i])):
|
2014-09-24 16:48:57 -04:00
|
|
|
buffer.pop(i)
|
|
|
|
|
2017-09-30 20:44:01 -04:00
|
|
|
if i == len(buffer) and previous_version_exists and not buffer[-1].strip().endswith(","):
|
2014-09-24 16:48:57 -04:00
|
|
|
# add comma
|
|
|
|
buffer[-1] = buffer[-1].rstrip() + ",\n"
|
|
|
|
|
2017-09-30 20:44:01 -04:00
|
|
|
if previous_version_exists:
|
|
|
|
last = buffer[-1]
|
|
|
|
spaces = ' ' * (len(last) - len(last.lstrip()))
|
|
|
|
else:
|
2017-09-30 23:20:51 -04:00
|
|
|
spaces = ' '
|
2014-09-24 16:48:57 -04:00
|
|
|
for (j, t) in enumerate(types):
|
2017-09-30 20:44:01 -04:00
|
|
|
if t == 'sorted':
|
|
|
|
newline = spaces + ('"sorted.%s"') % index_version
|
|
|
|
else:
|
|
|
|
newline = spaces + ('"%s-%s"' % (index_version, t))
|
2014-09-24 16:48:57 -04:00
|
|
|
if j < len(types) - 1 or i < len(buffer):
|
|
|
|
newline += ','
|
|
|
|
buffer.insert(i, newline + '\n')
|
|
|
|
i += 1
|
|
|
|
|
|
|
|
buffer.append(line)
|
|
|
|
return True
|
|
|
|
|
2016-06-25 14:26:42 -04:00
|
|
|
if 'Names = {' in line:
|
2014-09-24 16:48:57 -04:00
|
|
|
self.start = len(buffer) # location of first index name
|
|
|
|
buffer.append(line)
|
|
|
|
return False
|
|
|
|
|
|
|
|
changed = scriptutil.update_file(filename, matcher, Edit())
|
|
|
|
print('done' if changed else 'uptodate')
|
|
|
|
|
|
|
|
def check_backcompat_tests():
|
|
|
|
print(' checking backcompat tests...', end='', flush=True)
|
|
|
|
olddir = os.getcwd()
|
|
|
|
os.chdir('lucene/backward-codecs')
|
|
|
|
scriptutil.run('ant test -Dtestcase=TestBackwardsCompatibility')
|
|
|
|
os.chdir(olddir)
|
|
|
|
print('ok')
|
|
|
|
|
|
|
|
def download_from_mirror(version, remotename, localname):
|
|
|
|
url = 'http://apache.cs.utah.edu/lucene/java/%s/%s' % (version, remotename)
|
|
|
|
try:
|
|
|
|
urllib.request.urlretrieve(url, localname)
|
|
|
|
return True
|
|
|
|
except urllib.error.URLError as e:
|
|
|
|
if e.code == 404:
|
|
|
|
return False
|
|
|
|
raise e
|
|
|
|
|
|
|
|
def download_from_archives(version, remotename, localname):
|
|
|
|
url = 'http://archive.apache.org/dist/lucene/java/%s/%s' % (version, remotename)
|
|
|
|
try:
|
|
|
|
urllib.request.urlretrieve(url, localname)
|
|
|
|
return True
|
|
|
|
except urllib.error.URLError as e:
|
|
|
|
if e.code == 404:
|
|
|
|
return False
|
|
|
|
raise e
|
|
|
|
|
|
|
|
def download_release(version, temp_dir, force):
|
|
|
|
print(' downloading %s source release...' % version, end='', flush=True)
|
|
|
|
source = os.path.join(temp_dir, 'lucene-%s' % version)
|
|
|
|
if os.path.exists(source):
|
|
|
|
if force:
|
|
|
|
shutil.rmtree(source)
|
|
|
|
else:
|
|
|
|
print('uptodate')
|
|
|
|
return source
|
|
|
|
|
|
|
|
filename = 'lucene-%s-src.tgz' % version
|
|
|
|
source_tgz = os.path.join(temp_dir, filename)
|
|
|
|
if not download_from_mirror(version, filename, source_tgz) and \
|
|
|
|
not download_from_archives(version, filename, source_tgz):
|
|
|
|
raise Exception('Could not find version %s in apache mirror or archives' % version)
|
|
|
|
|
|
|
|
olddir = os.getcwd()
|
|
|
|
os.chdir(temp_dir)
|
|
|
|
scriptutil.run('tar -xvzf %s' % source_tgz)
|
|
|
|
os.chdir(olddir)
|
|
|
|
print('done')
|
|
|
|
return source
|
|
|
|
|
|
|
|
def read_config():
|
2015-04-15 17:36:00 -04:00
|
|
|
parser = argparse.ArgumentParser(formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
|
|
description='''\
|
|
|
|
Add backcompat index and test for new version. See:
|
|
|
|
http://wiki.apache.org/lucene-java/ReleaseTodo#Generate_Backcompat_Indexes
|
|
|
|
''')
|
2014-09-24 16:48:57 -04:00
|
|
|
parser.add_argument('--force', action='store_true', default=False,
|
|
|
|
help='Redownload the version and rebuild, even if it already exists')
|
|
|
|
parser.add_argument('--no-cleanup', dest='cleanup', action='store_false', default=True,
|
|
|
|
help='Do not cleanup the built indexes, so that they can be reused ' +
|
|
|
|
'for adding to another branch')
|
|
|
|
parser.add_argument('--temp-dir', metavar='DIR', default='/tmp/lucenebwc',
|
|
|
|
help='Temp directory to build backcompat indexes within')
|
|
|
|
parser.add_argument('version', type=scriptutil.Version.parse,
|
|
|
|
help='Version to add, of the form X.Y.Z')
|
|
|
|
c = parser.parse_args()
|
|
|
|
|
|
|
|
return c
|
|
|
|
|
|
|
|
def main():
|
|
|
|
c = read_config()
|
|
|
|
if not os.path.exists(c.temp_dir):
|
|
|
|
os.makedirs(c.temp_dir)
|
|
|
|
|
|
|
|
print('\nCreating backwards compatibility indexes')
|
|
|
|
source = download_release(c.version, c.temp_dir, c.force)
|
2016-06-25 14:26:42 -04:00
|
|
|
current_version = scriptutil.Version.parse(scriptutil.find_current_version())
|
|
|
|
create_and_add_index(source, 'cfs', c.version, current_version, c.temp_dir)
|
|
|
|
create_and_add_index(source, 'nocfs', c.version, current_version, c.temp_dir)
|
2017-10-26 17:11:54 -04:00
|
|
|
should_make_sorted = current_version.is_back_compat_with(c.version) \
|
|
|
|
and (c.version.major > 6 or (c.version.major == 6 and c.version.minor >= 2))
|
|
|
|
if should_make_sorted:
|
2017-10-24 12:27:55 -04:00
|
|
|
create_and_add_index(source, 'sorted', c.version, current_version, c.temp_dir)
|
2017-09-30 23:22:46 -04:00
|
|
|
if c.version.minor == 0 and c.version.bugfix == 0 and c.version.major < current_version.major:
|
2017-09-30 20:44:01 -04:00
|
|
|
create_and_add_index(source, 'moreterms', c.version, current_version, c.temp_dir)
|
|
|
|
create_and_add_index(source, 'dvupdates', c.version, current_version, c.temp_dir)
|
|
|
|
create_and_add_index(source, 'emptyIndex', c.version, current_version, c.temp_dir)
|
|
|
|
print ('\nMANUAL UPDATE REQUIRED: edit TestBackwardsCompatibility to enable moreterms, dvupdates, and empty index testing')
|
2014-09-24 16:48:57 -04:00
|
|
|
|
|
|
|
print('\nAdding backwards compatibility tests')
|
2016-06-25 14:26:42 -04:00
|
|
|
update_backcompat_tests(['cfs', 'nocfs'], c.version, current_version)
|
2017-10-26 17:11:54 -04:00
|
|
|
if should_make_sorted:
|
2017-10-24 12:27:55 -04:00
|
|
|
update_backcompat_tests(['sorted'], c.version, current_version)
|
2014-09-24 16:48:57 -04:00
|
|
|
|
|
|
|
print('\nTesting changes')
|
|
|
|
check_backcompat_tests()
|
|
|
|
|
|
|
|
if c.cleanup:
|
|
|
|
print('\nCleaning up')
|
|
|
|
print(' deleting %s...' % c.temp_dir, end='', flush=True)
|
|
|
|
shutil.rmtree(c.temp_dir)
|
|
|
|
print('done')
|
|
|
|
|
|
|
|
print()
|
|
|
|
|
|
|
|
if __name__ == '__main__':
|
|
|
|
try:
|
|
|
|
main()
|
|
|
|
except KeyboardInterrupt:
|
|
|
|
print('\nRecieved Ctrl-C, exiting early')
|