From 814d35e437511febd93339583a802b55b7e1b78d Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Thu, 1 Dec 2011 12:12:36 +0000 Subject: [PATCH] LUCENE-3612: remove _X.fnx git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1209047 13f79535-47bb-0310-9956-ffa450edef68 --- .../apache/lucene/index/IndexFileNames.java | 4 - .../org/apache/lucene/index/IndexWriter.java | 2 +- .../org/apache/lucene/index/SegmentInfos.java | 134 +---- .../codecs/DefaultSegmentInfosReader.java | 3 - .../codecs/DefaultSegmentInfosWriter.java | 1 - .../SimpleTextSegmentInfosReader.java | 4 - .../SimpleTextSegmentInfosWriter.java | 6 - .../apache/lucene/index/TestAddIndexes.java | 2 +- .../index/TestBackwardsCompatibility.java | 3 +- .../index/TestConsistentFieldNumbers.java | 9 - .../lucene/index/TestGlobalFieldNumbers.java | 529 ------------------ .../index/TestIndexWriterExceptions.java | 2 - 12 files changed, 9 insertions(+), 690 deletions(-) delete mode 100644 lucene/src/test/org/apache/lucene/index/TestGlobalFieldNumbers.java diff --git a/lucene/src/java/org/apache/lucene/index/IndexFileNames.java b/lucene/src/java/org/apache/lucene/index/IndexFileNames.java index d42e830b700..b7b6503d6cd 100644 --- a/lucene/src/java/org/apache/lucene/index/IndexFileNames.java +++ b/lucene/src/java/org/apache/lucene/index/IndexFileNames.java @@ -68,9 +68,6 @@ public final class IndexFileNames { /** Extension of separate norms */ public static final String SEPARATE_NORMS_EXTENSION = "s"; - /** Extension of global field numbers */ - public static final String GLOBAL_FIELD_NUM_MAP_EXTENSION = "fnx"; - /** * This array contains all filename extensions used by * Lucene's index files, with one exception, namely the @@ -85,7 +82,6 @@ public final class IndexFileNames { GEN_EXTENSION, NORMS_EXTENSION, COMPOUND_FILE_STORE_EXTENSION, - GLOBAL_FIELD_NUM_MAP_EXTENSION, }; public static final String[] NON_STORE_INDEX_EXTENSIONS = new String[] { diff --git a/lucene/src/java/org/apache/lucene/index/IndexWriter.java b/lucene/src/java/org/apache/lucene/index/IndexWriter.java index cb6bcff3d3b..9a632832660 100644 --- a/lucene/src/java/org/apache/lucene/index/IndexWriter.java +++ b/lucene/src/java/org/apache/lucene/index/IndexWriter.java @@ -882,7 +882,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { rollbackSegments = segmentInfos.createBackupSegmentInfos(true); // start with previous field numbers, but new FieldInfos - globalFieldNumberMap = segmentInfos.getOrLoadGlobalFieldNumberMap(directory); + globalFieldNumberMap = segmentInfos.getOrLoadGlobalFieldNumberMap(); docWriter = new DocumentsWriter(codec, config, directory, this, globalFieldNumberMap, bufferedDeletesStream); // Default deleter (for backwards compatibility) is diff --git a/lucene/src/java/org/apache/lucene/index/SegmentInfos.java b/lucene/src/java/org/apache/lucene/index/SegmentInfos.java index ae1972d7fb9..250f6613205 100644 --- a/lucene/src/java/org/apache/lucene/index/SegmentInfos.java +++ b/lucene/src/java/org/apache/lucene/index/SegmentInfos.java @@ -73,11 +73,6 @@ public final class SegmentInfos implements Cloneable, Iterable { */ public long version = System.currentTimeMillis(); - private long globalFieldMapVersion = 0; // version of the GFNM for the next commit - private long lastGlobalFieldMapVersion = 0; // version of the GFNM file we last successfully read or wrote - private long pendingMapVersion = -1; // version of the GFNM itself that we have last successfully written - // or -1 if we it was not written. This is set during prepareCommit - private long generation = 0; // generation of the "segments_N" for the next commit private long lastGeneration = 0; // generation of the "segments_N" file we last successfully read // or wrote; this is normally the same as generation except if @@ -184,15 +179,6 @@ public final class SegmentInfos implements Cloneable, Iterable { lastGeneration); } - private String getGlobalFieldNumberName(long version) { - /* - * This creates a file name ${version}.fnx without a leading underscore - * since this file might belong to more than one segment (global map) and - * could otherwise easily be confused with a per-segment file. - */ - return IndexFileNames.segmentFileName("_"+ version, "", IndexFileNames.GLOBAL_FIELD_NUM_MAP_EXTENSION); - } - /** * Parse the generation off the segments file name and * return it. @@ -297,8 +283,6 @@ public final class SegmentInfos implements Cloneable, Iterable { return null; } }.run(); - // either we are on 4.0 or we don't have a lastGlobalFieldMapVersion i.e. its still set to 0 - assert DefaultSegmentInfosWriter.FORMAT_4_0 <= format || (DefaultSegmentInfosWriter.FORMAT_4_0 > format && lastGlobalFieldMapVersion == 0); } // Only non-null after prepareCommit has been called and @@ -308,14 +292,6 @@ public final class SegmentInfos implements Cloneable, Iterable { private void write(Directory directory, Codec codec) throws IOException { String segmentFileName = getNextSegmentFileName(); - final String globalFieldMapFile; - if (globalFieldNumberMap != null && globalFieldNumberMap.isDirty()) { - globalFieldMapFile = getGlobalFieldNumberName(++globalFieldMapVersion); - pendingMapVersion = writeGlobalFieldMap(globalFieldNumberMap, directory, globalFieldMapFile); - } else { - globalFieldMapFile = null; - } - // Always advance the generation on write: if (generation == -1) { @@ -347,16 +323,6 @@ public final class SegmentInfos implements Cloneable, Iterable { } catch (Throwable t) { // Suppress so we keep throwing the original exception } - if (globalFieldMapFile != null) { // delete if written here - try { - // Try not to leave global field map in - // the index: - directory.deleteFile(globalFieldMapFile); - } catch (Throwable t) { - // Suppress so we keep throwing the original exception - } - } - pendingMapVersion = -1; } } } @@ -767,8 +733,6 @@ public final class SegmentInfos implements Cloneable, Iterable { void updateGeneration(SegmentInfos other) { lastGeneration = other.lastGeneration; generation = other.generation; - lastGlobalFieldMapVersion = other.lastGlobalFieldMapVersion; - globalFieldMapVersion = other.globalFieldMapVersion; } final void rollbackCommit(Directory dir) throws IOException { @@ -792,16 +756,6 @@ public final class SegmentInfos implements Cloneable, Iterable { // in our caller } pendingSegnOutput = null; - if (pendingMapVersion != -1) { - try { - final String fieldMapName = getGlobalFieldNumberName(globalFieldMapVersion--); - dir.deleteFile(fieldMapName); - } catch (Throwable t) { - // Suppress so we keep throwing the original exception - // in our caller - } - pendingMapVersion = -1; - } } } @@ -820,44 +774,6 @@ public final class SegmentInfos implements Cloneable, Iterable { throw new IllegalStateException("prepareCommit was already called"); write(dir, codec); } - - private final long writeGlobalFieldMap(FieldNumberBiMap map, Directory dir, String name) throws IOException { - final IndexOutput output = dir.createOutput(name, IOContext.READONCE); - boolean success = false; - long version; - try { - version = map.write(output); - success = true; - } finally { - try { - output.close(); - } catch (Throwable t) { - // throw orig excp - } - if (!success) { - try { - dir.deleteFile(name); - } catch (Throwable t) { - // throw orig excp - } - } else { - // we must sync here explicitly since during a commit - // IW will not sync the global field map. - dir.sync(Collections.singleton(name)); - } - } - return version; - } - - private void readGlobalFieldMap(FieldNumberBiMap map, Directory dir) throws IOException { - final String name = getGlobalFieldNumberName(lastGlobalFieldMapVersion); - final IndexInput input = dir.openInput(name, IOContext.READONCE); - try { - map.read(input); - } finally { - input.close(); - } - } /** Returns all file names referenced by SegmentInfo * instances matching the provided Directory (ie files @@ -876,9 +792,6 @@ public final class SegmentInfos implements Cloneable, Iterable { files.add(segmentFileName); } } - if (lastGlobalFieldMapVersion > 0) { - files.add(getGlobalFieldNumberName(lastGlobalFieldMapVersion)); - } final int size = size(); for(int i=0;i { } lastGeneration = generation; - if (pendingMapVersion != -1) { - /* - * TODO is it possible that the commit does not succeed here? if another - * commit happens at the same time and we lost the race between the - * prepareCommit and finishCommit the latest version is already - * incremented. - */ - globalFieldNumberMap.commitLastVersion(pendingMapVersion); - pendingMapVersion = -1; - lastGlobalFieldMapVersion = globalFieldMapVersion; - } try { IndexOutput genOutput = dir.createOutput(IndexFileNames.SEGMENTS_GEN, IOContext.READONCE); @@ -1003,7 +905,6 @@ public final class SegmentInfos implements Cloneable, Iterable { void replace(SegmentInfos other) { rollbackSegmentInfos(other.asList()); lastGeneration = other.lastGeneration; - lastGlobalFieldMapVersion = other.lastGlobalFieldMapVersion; format = other.format; } @@ -1027,47 +928,24 @@ public final class SegmentInfos implements Cloneable, Iterable { * Loads or returns the already loaded the global field number map for this {@link SegmentInfos}. * If this {@link SegmentInfos} has no global field number map the returned instance is empty */ - FieldNumberBiMap getOrLoadGlobalFieldNumberMap(Directory dir) throws IOException { + FieldNumberBiMap getOrLoadGlobalFieldNumberMap() throws IOException { if (globalFieldNumberMap != null) { return globalFieldNumberMap; } final FieldNumberBiMap map = new FieldNumberBiMap(); - if (lastGlobalFieldMapVersion > 0) { - // if we don't have a global map or this is a SI from a earlier version we just return the empty map; - readGlobalFieldMap(map, dir); - } if (size() > 0) { - if (format > DefaultSegmentInfosWriter.FORMAT_4_0) { - assert lastGlobalFieldMapVersion == 0; - // build the map up if we open a pre 4.0 index - for (SegmentInfo info : this) { - final FieldInfos segFieldInfos = info.getFieldInfos(); - for (FieldInfo fi : segFieldInfos) { - map.addOrGet(fi.name, fi.number); - } + // build the map up + for (SegmentInfo info : this) { + final FieldInfos segFieldInfos = info.getFieldInfos(); + for (FieldInfo fi : segFieldInfos) { + map.addOrGet(fi.name, fi.number); } } } return globalFieldNumberMap = map; } - /** - * Called by {@link SegmentInfosReader} when reading the global field map version - */ - public void setGlobalFieldMapVersion(long version) { - lastGlobalFieldMapVersion = globalFieldMapVersion = version; - } - - public long getGlobalFieldMapVersion() { - return globalFieldMapVersion; - } - - // for testing - long getLastGlobalFieldMapVersion() { - return lastGlobalFieldMapVersion; - } - /** applies all changes caused by committing a merge to this SegmentInfos */ void applyMergeChanges(MergePolicy.OneMerge merge, boolean dropSegment) { final Set mergedAway = new HashSet(merge.segments); diff --git a/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosReader.java b/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosReader.java index 534666bd428..0369cc7e8de 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosReader.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosReader.java @@ -49,9 +49,6 @@ public class DefaultSegmentInfosReader extends SegmentInfosReader { infos.version = input.readLong(); // read version infos.counter = input.readInt(); // read counter final int format = infos.getFormat(); - if (format <= DefaultSegmentInfosWriter.FORMAT_4_0) { - infos.setGlobalFieldMapVersion(input.readLong()); - } for (int i = input.readInt(); i > 0; i--) { // read segmentInfos SegmentInfo si = readSegmentInfo(directory, format, input); if (si.getVersion() == null) { diff --git a/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosWriter.java b/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosWriter.java index b16dba615bd..8198ac290fc 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosWriter.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosWriter.java @@ -68,7 +68,6 @@ public class DefaultSegmentInfosWriter extends SegmentInfosWriter { out.writeString(codecID); // write codecID out.writeLong(infos.version); out.writeInt(infos.counter); // write counter - out.writeLong(infos.getGlobalFieldMapVersion()); out.writeInt(infos.size()); // write infos for (SegmentInfo si : infos) { writeInfo(out, si); diff --git a/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextSegmentInfosReader.java b/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextSegmentInfosReader.java index 9b7bf4fbc83..917d698f5e2 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextSegmentInfosReader.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextSegmentInfosReader.java @@ -56,10 +56,6 @@ public class SimpleTextSegmentInfosReader extends SegmentInfosReader { assert StringHelper.startsWith(scratch, COUNTER); infos.counter = Integer.parseInt(readString(COUNTER.length, scratch)); - SimpleTextUtil.readLine(input, scratch); - assert StringHelper.startsWith(scratch, FNX_VERSION); - infos.setGlobalFieldMapVersion(Long.parseLong(readString(FNX_VERSION.length, scratch))); - SimpleTextUtil.readLine(input, scratch); assert StringHelper.startsWith(scratch, NUM_USERDATA); int numUserData = Integer.parseInt(readString(NUM_USERDATA.length, scratch)); diff --git a/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextSegmentInfosWriter.java b/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextSegmentInfosWriter.java index bb08421c4b9..e407e532a1c 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextSegmentInfosWriter.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextSegmentInfosWriter.java @@ -43,7 +43,6 @@ public class SimpleTextSegmentInfosWriter extends SegmentInfosWriter { final static BytesRef VERSION = new BytesRef("version "); final static BytesRef COUNTER = new BytesRef("counter "); - final static BytesRef FNX_VERSION = new BytesRef("global field map version "); final static BytesRef NUM_USERDATA = new BytesRef("user data entries "); final static BytesRef USERDATA_KEY = new BytesRef(" key "); final static BytesRef USERDATA_VALUE = new BytesRef(" value "); @@ -87,11 +86,6 @@ public class SimpleTextSegmentInfosWriter extends SegmentInfosWriter { SimpleTextUtil.write(out, COUNTER); SimpleTextUtil.write(out, Integer.toString(infos.counter), scratch); SimpleTextUtil.writeNewline(out); - - // global field map version - SimpleTextUtil.write(out, FNX_VERSION); - SimpleTextUtil.write(out, Long.toString(infos.getGlobalFieldMapVersion()), scratch); - SimpleTextUtil.writeNewline(out); // user data int numUserDataEntries = infos.getUserData() == null ? 0 : infos.getUserData().size(); diff --git a/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java b/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java index 636d6bb0572..ca846541dff 100755 --- a/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java +++ b/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java @@ -1082,7 +1082,7 @@ public class TestAddIndexes extends LuceneTestCase { w3.close(); // we should now see segments_X, // segments.gen,_Y.cfs,_Y.cfe, _Z.fnx - assertEquals("Only one compound segment should exist, but got: " + Arrays.toString(dir.listAll()), 5, dir.listAll().length); + assertEquals("Only one compound segment should exist, but got: " + Arrays.toString(dir.listAll()), 4, dir.listAll().length); dir.close(); } diff --git a/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java b/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java index 62e07751e1a..a4feb22dfcb 100644 --- a/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java +++ b/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java @@ -583,8 +583,7 @@ public class TestBackwardsCompatibility extends LuceneTestCase { "_0_1.del", "_0_1.s" + contentFieldIndex, "segments_2", - "segments.gen", - "_1.fnx"}; + "segments.gen"}; String[] actual = dir.listAll(); Arrays.sort(expected); diff --git a/lucene/src/test/org/apache/lucene/index/TestConsistentFieldNumbers.java b/lucene/src/test/org/apache/lucene/index/TestConsistentFieldNumbers.java index 5bd104be244..c4d258685dd 100644 --- a/lucene/src/test/org/apache/lucene/index/TestConsistentFieldNumbers.java +++ b/lucene/src/test/org/apache/lucene/index/TestConsistentFieldNumbers.java @@ -179,7 +179,6 @@ public class TestConsistentFieldNumbers extends LuceneTestCase { FieldInfos fis1 = sis.info(0).getFieldInfos(); assertEquals("f1", fis1.fieldInfo(0).name); assertEquals("f2", fis1.fieldInfo(1).name); - assertTrue(dir.fileExists("_1.fnx")); } @@ -203,8 +202,6 @@ public class TestConsistentFieldNumbers extends LuceneTestCase { assertEquals("f1", fis2.fieldInfo(0).name); assertNull(fis2.fieldInfo(1)); assertEquals("f3", fis2.fieldInfo(2).name); - assertFalse(dir.fileExists("_1.fnx")); - assertTrue(dir.fileExists("_2.fnx")); } { @@ -232,9 +229,6 @@ public class TestConsistentFieldNumbers extends LuceneTestCase { assertEquals("f1", fis3.fieldInfo(0).name); assertEquals("f2", fis3.fieldInfo(1).name); assertEquals("f3", fis3.fieldInfo(2).name); - assertFalse(dir.fileExists("_1.fnx")); - assertTrue(dir.fileExists("_2.fnx")); - assertFalse(dir.fileExists("_3.fnx")); } { @@ -262,9 +256,6 @@ public class TestConsistentFieldNumbers extends LuceneTestCase { assertEquals("f1", fis1.fieldInfo(0).name); assertEquals("f2", fis1.fieldInfo(1).name); assertEquals("f3", fis1.fieldInfo(2).name); - assertFalse(dir.fileExists("_1.fnx")); - assertTrue(dir.fileExists("_2.fnx")); - assertFalse(dir.fileExists("_3.fnx")); dir.close(); } } diff --git a/lucene/src/test/org/apache/lucene/index/TestGlobalFieldNumbers.java b/lucene/src/test/org/apache/lucene/index/TestGlobalFieldNumbers.java deleted file mode 100644 index eb3164f1b41..00000000000 --- a/lucene/src/test/org/apache/lucene/index/TestGlobalFieldNumbers.java +++ /dev/null @@ -1,529 +0,0 @@ -package org.apache.lucene.index; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import java.io.File; -import java.io.IOException; -import java.util.Collection; -import java.util.HashSet; -import java.util.List; -import java.util.Map.Entry; -import java.util.Set; -import java.util.SortedMap; -import java.util.TreeMap; - -import org.apache.lucene.analysis.MockAnalyzer; -import org.apache.lucene.document.BinaryField; -import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field; -import org.apache.lucene.document.FieldType; -import org.apache.lucene.document.TextField; -import org.apache.lucene.index.FieldInfos.FieldNumberBiMap; -import org.apache.lucene.index.codecs.DefaultSegmentInfosWriter; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.LockObtainFailedException; -import org.apache.lucene.util.FailOnNonBulkMergesInfoStream; -import org.apache.lucene.util.LuceneTestCase; -import org.apache.lucene.util._TestUtil; - -public class TestGlobalFieldNumbers extends LuceneTestCase { - - public void testGlobalFieldNumberFiles() throws IOException { - int num = atLeast(3); - for (int i = 0; i < num; i++) { - Directory dir = newDirectory(); - { - IndexWriterConfig config = newIndexWriterConfig(TEST_VERSION_CURRENT, - new MockAnalyzer(random)); - IndexWriter writer = new IndexWriter(dir, config); - Document d = new Document(); - d.add(new Field("f1", "d1 first field", TextField.TYPE_STORED)); - d.add(new Field("f2", "d1 second field", TextField.TYPE_STORED)); - writer.addDocument(d); - for (String string : writer.getIndexFileNames()) { - assertFalse(string.endsWith(".fnx")); - } - writer.commit(); - Collection files = writer.getIndexFileNames(); - files.remove("_1.fnx"); - for (String string : files) { - assertFalse(string.endsWith(".fnx")); - } - - assertFNXFiles(dir, "_1.fnx"); - d = new Document(); - d.add(new Field("f1", "d2 first field", TextField.TYPE_STORED)); - d.add(new BinaryField("f3", new byte[] { 1, 2, 3 })); - writer.addDocument(d); - writer.commit(); - files = writer.getIndexFileNames(); - files.remove("_2.fnx"); - for (String string : files) { - assertFalse(string.endsWith(".fnx")); - } - writer.close(); - assertFNXFiles(dir, "_2.fnx"); - } - - { - IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( - TEST_VERSION_CURRENT, new MockAnalyzer(random))); - Document d = new Document(); - d.add(new Field("f1", "d3 first field", TextField.TYPE_STORED)); - d.add(new Field("f2", "d3 second field", TextField.TYPE_STORED)); - d.add(new BinaryField("f3", new byte[] { 1, 2, 3, 4, 5 })); - writer.addDocument(d); - writer.close(); - Collection files = writer.getIndexFileNames(); - files.remove("_2.fnx"); - for (String string : files) { - assertFalse(string.endsWith(".fnx")); - } - - assertFNXFiles(dir, "_2.fnx"); - } - - IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( - TEST_VERSION_CURRENT, new MockAnalyzer(random)).setInfoStream(new FailOnNonBulkMergesInfoStream())); - writer.forceMerge(1); - writer.close(); - assertFNXFiles(dir, "_2.fnx"); - - dir.close(); - } - } - - public void testIndexReaderCommit() throws IOException { - int num = atLeast(3); - for (int i = 0; i < num; i++) { - Directory dir = newDirectory(); - { - IndexWriterConfig config = newIndexWriterConfig(TEST_VERSION_CURRENT, - new MockAnalyzer(random)); - IndexWriter writer = new IndexWriter(dir, config); - Document d = new Document(); - d.add(new Field("f1", "d1 first field", TextField.TYPE_STORED)); - d.add(new Field("f2", "d1 second field", TextField.TYPE_STORED)); - writer.addDocument(d); - writer.commit(); - assertFNXFiles(dir, "_1.fnx"); - d = new Document(); - d.add(new Field("f1", "d2 first field", TextField.TYPE_STORED)); - d.add(new BinaryField("f3", new byte[] { 1, 2, 3 })); - writer.addDocument(d); - writer.commit(); - writer.close(); - assertFNXFiles(dir, "_2.fnx"); - } - IndexReader reader = IndexReader.open(dir, false); - reader.deleteDocument(0); - reader.commit(); - reader.close(); - // make sure this reader can not modify the field map - assertFNXFiles(dir, "_2.fnx"); - - IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( - TEST_VERSION_CURRENT, new MockAnalyzer(random)).setInfoStream(new FailOnNonBulkMergesInfoStream())); - writer.forceMerge(1); - writer.close(); - assertFNXFiles(dir, "_2.fnx"); - - dir.close(); - } - } - - public void testGlobalFieldNumberFilesAcrossCommits() throws IOException { - int num = atLeast(3); - for (int i = 0; i < num; i++) { - Directory dir = newDirectory(); - { - IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( - TEST_VERSION_CURRENT, new MockAnalyzer(random)).setIndexDeletionPolicy( - new KeepAllDeletionPolicy())); - Document d = new Document(); - d.add(new Field("f1", "d1 first field", TextField.TYPE_STORED)); - d.add(new Field("f2", "d1 second field", TextField.TYPE_STORED)); - writer.addDocument(d); - writer.commit(); - assertFNXFiles(dir, "_1.fnx"); - d = new Document(); - d.add(new Field("f1", "d2 first field", TextField.TYPE_STORED)); - d.add(new BinaryField("f3", new byte[] { 1, 2, 3 })); - writer.addDocument(d); - writer.commit(); - writer.commit(); - writer.commit(); - assertFNXFiles(dir, "_1.fnx", "_2.fnx"); - writer.close(); - assertFNXFiles(dir, "_1.fnx", "_2.fnx"); - } - - { - IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( - TEST_VERSION_CURRENT, new MockAnalyzer(random))); - Document d = new Document(); - d.add(new Field("f1", "d3 first field", TextField.TYPE_STORED)); - d.add(new Field("f2", "d3 second field", TextField.TYPE_STORED)); - d.add(new BinaryField("f3", new byte[] { 1, 2, 3, 4, 5 })); - writer.addDocument(d); - writer.close(); - assertFNXFiles(dir, "_2.fnx"); - } - IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( - TEST_VERSION_CURRENT, new MockAnalyzer(random)).setInfoStream(new FailOnNonBulkMergesInfoStream())); - writer.forceMerge(1); - writer.close(); - assertFNXFiles(dir, "_2.fnx"); - dir.close(); - } - } - - public void testGlobalFieldNumberOnOldCommit() throws IOException { - int num = atLeast(3); - for (int i = 0; i < num; i++) { - Directory dir = newDirectory(); - IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( - TEST_VERSION_CURRENT, new MockAnalyzer(random)).setIndexDeletionPolicy( - new KeepAllDeletionPolicy())); - Document d = new Document(); - d.add(new Field("f1", "d1 first field", TextField.TYPE_STORED)); - d.add(new Field("f2", "d1 second field", TextField.TYPE_STORED)); - writer.addDocument(d); - writer.commit(); - assertFNXFiles(dir, "_1.fnx"); - d = new Document(); - d.add(new Field("f1", "d2 first field", TextField.TYPE_STORED)); - d.add(new BinaryField("f3", new byte[] { 1, 2, 3 })); - writer.addDocument(d); - assertFNXFiles(dir, "_1.fnx"); - writer.close(); - assertFNXFiles(dir, "_1.fnx", "_2.fnx"); - // open first commit - List listCommits = IndexReader.listCommits(dir); - assertEquals(2, listCommits.size()); - writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, - new MockAnalyzer(random)).setIndexDeletionPolicy( - new KeepAllDeletionPolicy()).setIndexCommit(listCommits.get(0))); - - d = new Document(); - d.add(new Field("f1", "d2 first field", TextField.TYPE_STORED)); - d.add(new BinaryField("f3", new byte[] { 1, 2, 3 })); - writer.addDocument(d); - writer.commit(); - // now we have 3 files since f3 is not present in the first commit - assertFNXFiles(dir, "_1.fnx", "_2.fnx", "_3.fnx"); - writer.close(); - assertFNXFiles(dir, "_1.fnx", "_2.fnx", "_3.fnx"); - - writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, - new MockAnalyzer(random))); - writer.commit(); - listCommits = IndexReader.listCommits(dir); - assertEquals(1, listCommits.size()); - assertFNXFiles(dir, "_3.fnx"); - writer.close(); - assertFNXFiles(dir, "_3.fnx"); - dir.close(); - } - } - - private final Directory buildRandomIndex(String[] fieldNames, int numDocs, - IndexWriterConfig conf) throws CorruptIndexException, - LockObtainFailedException, IOException { - Directory dir = newDirectory(); - IndexWriter writer = new IndexWriter(dir, conf); - for (int i = 0; i < numDocs; i++) { - Document doc = new Document(); - final int numFields = 1 + random.nextInt(fieldNames.length); - for (int j = 0; j < numFields; j++) { - FieldType customType = new FieldType(); - customType.setIndexed(true); - customType.setTokenized(random.nextBoolean()); - customType.setOmitNorms(random.nextBoolean()); - doc.add(newField(fieldNames[random.nextInt(fieldNames.length)], - _TestUtil.randomRealisticUnicodeString(random), - customType)); - - } - writer.addDocument(doc); - if (random.nextInt(20) == 0) { - writer.commit(); - } - } - writer.close(); - return dir; - } - - public void testForceMerge() throws IOException { - for (int i = 0; i < 2*RANDOM_MULTIPLIER; i++) { - Set fieldNames = new HashSet(); - final int numFields = 2 + (TEST_NIGHTLY ? random.nextInt(200) : random.nextInt(20)); - for (int j = 0; j < numFields; j++) { - fieldNames.add("field_" + j); - } - Directory base = buildRandomIndex(fieldNames.toArray(new String[0]), - 20 + random.nextInt(100), - newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))); - IndexWriter writer = new IndexWriter(base, newIndexWriterConfig( - TEST_VERSION_CURRENT, new MockAnalyzer(random))); - FieldNumberBiMap globalFieldMap = writer.segmentInfos - .getOrLoadGlobalFieldNumberMap(base); - Set> entries = globalFieldMap.entries(); - writer.forceMerge(1); - writer.commit(); - writer.close(); - Set> afterOptmize = globalFieldMap.entries(); - assertEquals(entries, afterOptmize); - base.close(); - } - } - - public void testAddIndexesStableFieldNumbers() throws IOException { - for (int i = 0; i < 2*RANDOM_MULTIPLIER; i++) { - Set fieldNames = new HashSet(); - final int numFields = 2 + (TEST_NIGHTLY ? random.nextInt(50) : random.nextInt(10)); - for (int j = 0; j < numFields; j++) { - fieldNames.add("field_" + j); - } - - Directory base = newDirectory(); - IndexWriter writer = new IndexWriter(base, newIndexWriterConfig( - TEST_VERSION_CURRENT, new MockAnalyzer(random))); - Document doc = new Document(); - for (String string : fieldNames) { - FieldType customType = new FieldType(); - customType.setIndexed(true); - customType.setTokenized(random.nextBoolean()); - customType.setOmitNorms(random.nextBoolean()); - doc.add(newField(string, - _TestUtil.randomRealisticUnicodeString(random), - customType)); - - } - writer.addDocument(doc); - writer.commit(); - FieldNumberBiMap globalFieldMap = writer.segmentInfos - .getOrLoadGlobalFieldNumberMap(base); - final Set> entries = globalFieldMap.entries(); - assertEquals(entries.size(), fieldNames.size()); - for (Entry entry : entries) { - // all fields are in this fieldMap - assertTrue(fieldNames.contains(entry.getKey())); - } - writer.close(); - - int numIndexes = 1 + random.nextInt(10); - for (int j = 0; j < numIndexes; j++) { - Directory toAdd = buildRandomIndex(fieldNames.toArray(new String[0]), - 1 + random.nextInt(50), - newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))); - IndexWriter w = new IndexWriter(base, newIndexWriterConfig( - TEST_VERSION_CURRENT, new MockAnalyzer(random))); - if (random.nextBoolean()) { - IndexReader open = IndexReader.open(toAdd); - w.addIndexes(open); - open.close(); - } else { - w.addIndexes(toAdd); - } - - w.close(); - FieldNumberBiMap map = w.segmentInfos - .getOrLoadGlobalFieldNumberMap(toAdd); - assertEquals(entries, map.entries()); - toAdd.close(); - } - IndexWriter w = new IndexWriter(base, newIndexWriterConfig( - TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMergePolicy( - new LogByteSizeMergePolicy())); - w.forceMerge(1); - w.close(); - SegmentInfos sis = new SegmentInfos(); - sis.read(base); - SegmentInfo segmentInfo = sis.info(sis.size() - 1);// last segment must - // have all fields with - // consistent numbers - FieldInfos fieldInfos = segmentInfo.getFieldInfos(); - assertEquals(fieldInfos.size(), entries.size()); - for (Entry entry : entries) { - assertEquals(entry.getValue(), - Integer.valueOf(fieldInfos.fieldNumber(entry.getKey()))); - assertEquals(entry.getKey(), fieldInfos.fieldName(entry.getValue())); - } - base.close(); - } - } - - final String[] oldNames = { "30.cfs", "30.nocfs", "31.cfs", "31.nocfs", }; - - public void testAddOldIndex() throws IOException { - int i = random.nextInt(oldNames.length); - File oldIndxeDir = _TestUtil.getTempDir(oldNames[i]); - Directory dir = null; - try { - _TestUtil - .unzip(getDataFile("index." + oldNames[i] + ".zip"), oldIndxeDir); - dir = newFSDirectory(oldIndxeDir); - SegmentInfos infos = new SegmentInfos(); - infos.read(dir); - SortedMap sortedMap = new TreeMap(); - - FieldNumberBiMap biMap = new FieldNumberBiMap(); - int maxFieldNum = Integer.MIN_VALUE; - for (SegmentInfo segmentInfo : infos) { - for (FieldInfo fieldInfo : segmentInfo.getFieldInfos()) { - int globNumber = biMap.addOrGet(fieldInfo.name, fieldInfo.number); - maxFieldNum = Math.max(maxFieldNum, globNumber); - sortedMap.put(globNumber, fieldInfo.name); - } - } - Directory base = newDirectory(); - IndexWriter writer = new IndexWriter(base, newIndexWriterConfig( - TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMergePolicy( - NoMergePolicy.NO_COMPOUND_FILES)); - - SortedMap copySortedMap = new TreeMap( - sortedMap); - while (!sortedMap.isEmpty()) { // add every field at least once - Document doc = new Document(); - int nextField = random.nextInt(maxFieldNum + 1); - sortedMap.remove(nextField); - - String name = copySortedMap.get(nextField); - assertNotNull(name); - - FieldType customType = new FieldType(); - customType.setIndexed(true); - customType.setTokenized(random.nextBoolean()); - customType.setOmitNorms(random.nextBoolean()); - doc.add(newField(name, _TestUtil.randomRealisticUnicodeString(random), - customType)); - writer.addDocument(doc); - if (random.nextInt(10) == 0) { - writer.commit(); - } - } - Set> expectedEntries = writer.segmentInfos - .getOrLoadGlobalFieldNumberMap(base).entries(); - writer.addIndexes(dir); // add the old index - writer.close(); - - writer = new IndexWriter(base, newIndexWriterConfig(TEST_VERSION_CURRENT, - new MockAnalyzer(random)).setMergePolicy(NoMergePolicy.NO_COMPOUND_FILES)); - writer.commit(); // make sure the old index is the latest segment - writer.close(); - - // we don't merge here since we use NoMergePolicy - SegmentInfos sis = new SegmentInfos(); - sis.read(base); - // check that the latest global field numbers are consistent and carried - // over from the 4.0 index - FieldNumberBiMap actualGlobalMap = sis - .getOrLoadGlobalFieldNumberMap(base); - assertEquals(expectedEntries, actualGlobalMap.entries()); - base.close(); - } finally { - if (dir != null) - dir.close(); - _TestUtil.rmDir(oldIndxeDir); - } - } - - public void testFilesOnOldIndex() throws IOException { - int i = random.nextInt(oldNames.length); - File oldIndxeDir = _TestUtil.getTempDir(oldNames[i]); - Directory dir = null; - - MergePolicy policy = random.nextBoolean() ? NoMergePolicy.COMPOUND_FILES : NoMergePolicy.NO_COMPOUND_FILES; - try { - _TestUtil - .unzip(getDataFile("index." + oldNames[i] + ".zip"), oldIndxeDir); - dir = newFSDirectory(oldIndxeDir); - IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( - TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMergePolicy(policy)); - SegmentInfos segmentInfos = writer.segmentInfos; - assertTrue(DefaultSegmentInfosWriter.FORMAT_4_0 < segmentInfos.getFormat()); - assertEquals(0, segmentInfos.getGlobalFieldMapVersion()); - for (String string : writer.getIndexFileNames()) { - assertFalse(string.endsWith(".fnx")); - } - writer.commit(); - - assertTrue(DefaultSegmentInfosWriter.FORMAT_4_0 < segmentInfos.getFormat()); - assertEquals(0, segmentInfos.getGlobalFieldMapVersion()); - Collection files = writer.getIndexFileNames(); - for (String string : files) { - assertFalse(string.endsWith(".fnx")); - } - - Document d = new Document(); - d.add(new Field("f1", "d1 first field", TextField.TYPE_STORED)); - writer.addDocument(d); - writer.prepareCommit(); - // the fnx file should still be under control of the SIS - assertTrue(DefaultSegmentInfosWriter.FORMAT_4_0 < segmentInfos.getFormat()); - assertEquals(0, segmentInfos.getLastGlobalFieldMapVersion()); - assertEquals(1, segmentInfos.getGlobalFieldMapVersion()); - files = writer.getIndexFileNames(); - for (String string : files) { - assertFalse(string.endsWith(".fnx")); - } - - writer.commit(); - - // now we should see the fnx file even if this is a 3.x segment - assertTrue(DefaultSegmentInfosWriter.FORMAT_4_0 < segmentInfos.getFormat()); - assertEquals(1, segmentInfos.getGlobalFieldMapVersion()); - assertEquals(1, segmentInfos.getLastGlobalFieldMapVersion()); - files = writer.getIndexFileNames(); - assertTrue(files.remove("_1.fnx")); - for (String string : files) { - assertFalse(string.endsWith(".fnx")); - } - writer.close(); - } finally { - if (dir != null) - dir.close(); - _TestUtil.rmDir(oldIndxeDir); - } - } - - class KeepAllDeletionPolicy implements IndexDeletionPolicy { - public void onInit(List commits) throws IOException { - } - - public void onCommit(List commits) - throws IOException { - } - } - - public static void assertFNXFiles(Directory dir, String... expectedFnxFiles) - throws IOException { - String[] listAll = dir.listAll(); - Set fnxFiles = new HashSet(); - for (String string : listAll) { - if (string.endsWith(".fnx")) { - fnxFiles.add(string); - } - } - assertEquals("" + fnxFiles, expectedFnxFiles.length, fnxFiles.size()); - for (String string : expectedFnxFiles) { - assertTrue(" missing fnx file: " + string, fnxFiles.contains(string)); - } - } - -} diff --git a/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java b/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java index 9162a9368af..88d7366b02a 100644 --- a/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java +++ b/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java @@ -916,10 +916,8 @@ public class TestIndexWriterExceptions extends LuceneTestCase { } catch (RuntimeException re) { // Expected } - assertTrue(dir.fileExists("_1.fnx")); assertTrue(failure.failOnCommit && failure.failOnDeleteFile); w.rollback(); - assertFalse(dir.fileExists("_1.fnx")); assertEquals(0, dir.listAll().length); dir.close(); }