Apache/lucene
1
0
Fork 0
mirror of https://github.com/apache/lucene.git synced 2025-02-21 01:18:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix bw index generation logic.

Browse Source
This commit is contained in:
Adrien Grand 2024-02-20 22:10:01 +01:00
parent 47b02dba1e
commit 13d561af1d
3 changed files with 25 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
dev-tools/scripts/addBackcompatIndexes.py
View File

@ -45,16 +45,13 @@ def create_and_add_index(source, indextype, index_version, current_version, temp
'emptyIndex': 'empty'
}[indextype]
if indextype in ('cfs', 'nocfs'):
dirname = 'index.%s' % indextype
filename = '%s.%s-%s.zip' % (prefix, index_version, indextype)
else:
dirname = indextype
filename = '%s.%s.zip' % (prefix, index_version)
print(' creating %s...' % filename, end='', flush=True)
module = 'backward-codecs'
index_dir = os.path.join('lucene', module, 'src/test/org/apache/lucene/backward_index')
test_file = os.path.join(index_dir, filename)
if os.path.exists(os.path.join(index_dir, filename)):
print('uptodate')
return
@ -76,24 +73,20 @@ def create_and_add_index(source, indextype, index_version, current_version, temp
'-Dtests.codec=default'
])
base_dir = os.getcwd()
bc_index_dir = os.path.join(temp_dir, dirname)
bc_index_file = os.path.join(bc_index_dir, filename)
bc_index_file = os.path.join(temp_dir, filename)
if os.path.exists(bc_index_file):
print('alreadyexists')
else:
if os.path.exists(bc_index_dir):
shutil.rmtree(bc_index_dir)
os.chdir(source)
scriptutil.run('./gradlew %s' % gradle_args)
os.chdir(bc_index_dir)
scriptutil.run('zip %s *' % filename)
if not os.path.exists(bc_index_file):
raise Exception("Expected file can't be found: %s" %bc_index_file)
print('done')
print(' adding %s...' % filename, end='', flush=True)
scriptutil.run('cp %s %s' % (bc_index_file, os.path.join(base_dir, index_dir)))
os.chdir(base_dir)
scriptutil.run('rm -rf %s' % bc_index_dir)
print('done')
def update_backcompat_tests(index_version, current_version):

23
lucene/backward-codecs/src/test/org/apache/lucene/backward_index/BackwardsCompatibilityTestBase.java
View File

@ -17,6 +17,7 @@
package org.apache.lucene.backward_index;
import com.carrotsearch.randomizedtesting.annotations.Name;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.LineNumberReader;
@ -38,11 +39,17 @@ import java.util.function.Predicate;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.SegmentReader;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.OutputStreamDataOutput;
import org.apache.lucene.tests.util.LuceneTestCase;
import org.apache.lucene.tests.util.TestUtil;
import org.apache.lucene.util.BytesRef;
@ -253,10 +260,20 @@ public abstract class BackwardsCompatibilityTestBase extends LuceneTestCase {
protected abstract void createIndex(Directory directory) throws IOException;
public final void createBWCIndex() throws IOException {
Path indexDir = getIndexDir().resolve(indexName(Version.LATEST));
Files.deleteIfExists(indexDir);
try (Directory dir = newFSDirectory(indexDir)) {
Path zipFile = getIndexDir().resolve(indexName(Version.LATEST));
Files.deleteIfExists(zipFile);
Path tmpDir = createTempDir();
try (Directory dir = FSDirectory.open(tmpDir);
ZipOutputStream zipOut = new ZipOutputStream(new FileOutputStream(zipFile.toFile()))) {
createIndex(dir);
for (String file : dir.listAll()) {
try (IndexInput in = dir.openInput(file, IOContext.READONCE)) {
zipOut.putNextEntry(new ZipEntry(file));
new OutputStreamDataOutput(zipOut).copyBytes(in, in.length());
zipOut.closeEntry();
}
}
}
}

2
lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestGenerateBwcIndices.java
View File

@ -20,8 +20,10 @@ import static org.apache.lucene.backward_index.BackwardsCompatibilityTestBase.cr
import java.io.IOException;
import org.apache.lucene.tests.util.LuceneTestCase;
import org.apache.lucene.tests.util.LuceneTestCase.SuppressFileSystems;
import org.apache.lucene.util.Version;
@SuppressFileSystems("ExtrasFS")
public class TestGenerateBwcIndices extends LuceneTestCase {
// Backcompat index generation, described below, is mostly automated in: