diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index c5a85d0c343..0de9da7b653 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -15,6 +15,14 @@ New Features long "sequence number" indicating the effective equivalent single-threaded execution order (Mike McCandless) +* LUCENE-7335: IndexWriter's commit data is now late binding, + recording key/values from a provided iterable based on when the + commit actually takes place (Mike McCandless) + +Bug Fixes + +* LUCENE-6662: Fixed potential resource leaks. (Rishabh Patel via Adrien Grand) + Improvements * LUCENE-7323: Compound file writing now verifies the incoming diff --git a/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CommitIndexTask.java b/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CommitIndexTask.java index fda49947c26..e09cfdfed3e 100644 --- a/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CommitIndexTask.java +++ b/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CommitIndexTask.java @@ -51,7 +51,7 @@ public class CommitIndexTask extends PerfTask { IndexWriter iw = getRunData().getIndexWriter(); if (iw != null) { if (commitUserData != null) { - iw.setCommitData(commitUserData); + iw.setLiveCommitData(commitUserData.entrySet()); } iw.commit(); } diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexCommit.java b/lucene/core/src/java/org/apache/lucene/index/IndexCommit.java index 7c3ed72b857..0c4d323621c 100644 --- a/lucene/core/src/java/org/apache/lucene/index/IndexCommit.java +++ b/lucene/core/src/java/org/apache/lucene/index/IndexCommit.java @@ -108,7 +108,7 @@ public abstract class IndexCommit implements Comparable { public abstract long getGeneration(); /** Returns userData, previously passed to {@link - * IndexWriter#setCommitData(Map)} for this commit. Map is + * IndexWriter#setLiveCommitData(Iterable)} for this commit. Map is * {@code String -> String}. */ public abstract Map getUserData() throws IOException; diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexUpgrader.java b/lucene/core/src/java/org/apache/lucene/index/IndexUpgrader.java index 02690c4bc6a..fffc3769da6 100644 --- a/lucene/core/src/java/org/apache/lucene/index/IndexUpgrader.java +++ b/lucene/core/src/java/org/apache/lucene/index/IndexUpgrader.java @@ -174,7 +174,7 @@ public final class IndexUpgrader { infoStream.message(LOG_PREFIX, "All segments upgraded to version " + Version.LATEST); infoStream.message(LOG_PREFIX, "Enforcing commit to rewrite all index metadata..."); } - w.setCommitData(w.getCommitData()); // fake change to enforce a commit (e.g. if index has no segments) + w.setLiveCommitData(w.getLiveCommitData()); // fake change to enforce a commit (e.g. if index has no segments) assert w.hasUncommittedChanges(); w.commit(); if (infoStream.isEnabled(LOG_PREFIX)) { diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java index 5fe164856ee..f0d756bdfb5 100644 --- a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java +++ b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java @@ -295,6 +295,8 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable { private volatile boolean closed; private volatile boolean closing; + private Iterable> commitUserData; + // Holds all SegmentInfo instances currently involved in // merges private HashSet mergingSegments = new HashSet<>(); @@ -947,6 +949,8 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable { rollbackSegments = segmentInfos.createBackupSegmentInfos(); } + commitUserData = new HashMap(segmentInfos.getUserData()).entrySet(); + pendingNumDocs.set(segmentInfos.totalMaxDoc()); // start with previous field numbers, but new FieldInfos @@ -2997,6 +3001,14 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable { segmentInfos.changed(); } + if (commitUserData != null) { + Map userData = new HashMap<>(); + for(Map.Entry ent : commitUserData) { + userData.put(ent.getKey(), ent.getValue()); + } + segmentInfos.setUserData(userData, false); + } + // Must clone the segmentInfos while we still // hold fullFlushLock and while sync'd so that // no partial changes (eg a delete w/o @@ -3011,7 +3023,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable { // we are trying to sync all referenced files, a // merge completes which would otherwise have // removed the files we are now syncing. - filesToCommit = toCommit.files(false); + filesToCommit = toCommit.files(false); deleter.incRef(filesToCommit); } success = true; @@ -3059,38 +3071,65 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable { } /** - * Sets the commit user data map. That method is considered a transaction by - * {@link IndexWriter} and will be {@link #commit() committed} even if no other - * changes were made to the writer instance. Note that you must call this method - * before {@link #prepareCommit()}, or otherwise it won't be included in the + * Sets the iterator to provide the commit user data map at commit time. Calling this method + * is considered a committable change and will be {@link #commit() committed} even if + * there are no other changes this writer. Note that you must call this method + * before {@link #prepareCommit()}. Otherwise it won't be included in the * follow-on {@link #commit()}. *

- * NOTE: the map is cloned internally, therefore altering the map's - * contents after calling this method has no effect. + * NOTE: the iterator is late-binding: it is only visited once all documents for the + * commit have been written to their segments, before the next segments_N file is written */ + public final synchronized void setLiveCommitData(Iterable> commitUserData) { + setLiveCommitData(commitUserData, true); + } + + /** + * Sets the commit user data map. + * + * @deprecated Use {@link #setLiveCommitData} instead. + */ + @Deprecated public final synchronized void setCommitData(Map commitUserData) { - setCommitData(commitUserData, true); + setLiveCommitData(new HashMap<>(commitUserData).entrySet()); } /** - * Sets the commit user data map, controlling whether to advance the {@link SegmentInfos#getVersion}. + * Sets the commit user data iterator, controlling whether to advance the {@link SegmentInfos#getVersion}. * - * @see #setCommitData(Map) + * @see #setLiveCommitData(Iterable) * * @lucene.internal */ - public final synchronized void setCommitData(Map commitUserData, boolean doIncrementVersion) { - segmentInfos.setUserData(new HashMap<>(commitUserData), doIncrementVersion); + public final synchronized void setLiveCommitData(Iterable> commitUserData, boolean doIncrementVersion) { + this.commitUserData = commitUserData; + if (doIncrementVersion) { + segmentInfos.changed(); + } changeCount.incrementAndGet(); } + /** + * Returns the commit user data iterable previously set with {@link #setLiveCommitData(Iterable)}, or null if nothing has been set yet. + */ + public final synchronized Iterable> getLiveCommitData() { + return commitUserData; + } + /** * Returns the commit user data map that was last committed, or the one that * was set on {@link #setCommitData(Map)}. + * + * @deprecated Use {@link #getLiveCommitData} instead. */ + @Deprecated public final synchronized Map getCommitData() { - return segmentInfos.getUserData(); + Map data = new HashMap<>(); + for(Map.Entry ent : commitUserData) { + data.put(ent.getKey(), ent.getValue()); + } + return data; } - + // Used only by commit and prepareCommit, below; lock // order is commitLock -> IW private final Object commitLock = new Object(); diff --git a/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java b/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java index 642b320cf37..5db339ec83b 100644 --- a/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java +++ b/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java @@ -17,6 +17,20 @@ package org.apache.lucene.index; +import java.io.IOException; +import java.io.PrintStream; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map.Entry; +import java.util.Map; +import java.util.Set; + import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.DocValuesFormat; @@ -32,20 +46,6 @@ import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.StringHelper; import org.apache.lucene.util.Version; -import java.io.IOException; -import java.io.PrintStream; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map.Entry; -import java.util.Map; -import java.util.Set; - /** * A collection of segmentInfo objects with methods for operating on those * segments in relation to the file system. @@ -103,7 +103,7 @@ import java.util.Set; *

  • SegID is the identifier of the Codec that encoded this segment.
    • *
  • CommitUserData stores an optional user-supplied opaque * Map<String,String> that was passed to - * {@link IndexWriter#setCommitData(java.util.Map)}.
    • + * {@link IndexWriter#setLiveCommitData(Iterable)}. *
  • FieldInfosGen is the generation count of the fieldInfos file. If this is * -1, there are no updates to the fieldInfos in that segment. Anything above * zero means there are updates to fieldInfos stored by {@link FieldInfosFormat} @@ -453,7 +453,7 @@ public final class SegmentInfos implements Cloneable, Iterable commitData = new HashMap<>(); commitData.put("commitTime", String.valueOf(System.currentTimeMillis())); - writer.setCommitData(commitData); + writer.setLiveCommitData(commitData.entrySet()); writer.commit(); writer.close(); @@ -251,7 +251,7 @@ public class TestDeletionPolicy extends LuceneTestCase { } commitData = new HashMap<>(); commitData.put("commitTime", String.valueOf(System.currentTimeMillis())); - writer.setCommitData(commitData); + writer.setLiveCommitData(commitData.entrySet()); writer.commit(); writer.close(); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReaderReopen.java b/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReaderReopen.java index e2102cb8f1b..0138b90b55c 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReaderReopen.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReaderReopen.java @@ -557,14 +557,14 @@ public class TestDirectoryReaderReopen extends LuceneTestCase { writer.addDocument(doc); Map data = new HashMap<>(); data.put("index", i+""); - writer.setCommitData(data); + writer.setLiveCommitData(data.entrySet()); writer.commit(); } for(int i=0;i<4;i++) { writer.deleteDocuments(new Term("id", ""+i)); Map data = new HashMap<>(); data.put("index", (4+i)+""); - writer.setCommitData(data); + writer.setLiveCommitData(data.entrySet()); writer.commit(); } writer.close(); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java index 34ee56f2725..7a47d973c38 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java @@ -1892,9 +1892,9 @@ public class TestIndexWriter extends LuceneTestCase { writer.commit(); // first commit to complete IW create transaction. // this should store the commit data, even though no other changes were made - writer.setCommitData(new HashMap() {{ + writer.setLiveCommitData(new HashMap() {{ put("key", "value"); - }}); + }}.entrySet()); writer.commit(); DirectoryReader r = DirectoryReader.open(dir); @@ -1902,13 +1902,13 @@ public class TestIndexWriter extends LuceneTestCase { r.close(); // now check setCommitData and prepareCommit/commit sequence - writer.setCommitData(new HashMap() {{ + writer.setLiveCommitData(new HashMap() {{ put("key", "value1"); - }}); + }}.entrySet()); writer.prepareCommit(); - writer.setCommitData(new HashMap() {{ + writer.setLiveCommitData(new HashMap() {{ put("key", "value2"); - }}); + }}.entrySet()); writer.commit(); // should commit the first commitData only, per protocol r = DirectoryReader.open(dir); @@ -1926,21 +1926,32 @@ public class TestIndexWriter extends LuceneTestCase { writer.close(); dir.close(); } + + private Map getLiveCommitData(IndexWriter writer) { + Map data = new HashMap<>(); + Iterable> iter = writer.getLiveCommitData(); + if (iter != null) { + for(Map.Entry ent : iter) { + data.put(ent.getKey(), ent.getValue()); + } + } + return data; + } @Test public void testGetCommitData() throws Exception { Directory dir = newDirectory(); IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(null)); - writer.setCommitData(new HashMap() {{ + writer.setLiveCommitData(new HashMap() {{ put("key", "value"); - }}); - assertEquals("value", writer.getCommitData().get("key")); + }}.entrySet()); + assertEquals("value", getLiveCommitData(writer).get("key")); writer.close(); // validate that it's also visible when opening a new IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(null) .setOpenMode(OpenMode.APPEND)); - assertEquals("value", writer.getCommitData().get("key")); + assertEquals("value", getLiveCommitData(writer).get("key")); writer.close(); dir.close(); @@ -2650,9 +2661,9 @@ public class TestIndexWriter extends LuceneTestCase { DirectoryReader r = DirectoryReader.open(w); Map m = new HashMap<>(); m.put("foo", "bar"); - w.setCommitData(m); + w.setLiveCommitData(m.entrySet()); - // setCommitData with no other changes should count as an NRT change: + // setLiveCommitData with no other changes should count as an NRT change: DirectoryReader r2 = DirectoryReader.openIfChanged(r); assertNotNull(r2); @@ -2669,9 +2680,9 @@ public class TestIndexWriter extends LuceneTestCase { DirectoryReader r = DirectoryReader.open(w); Map m = new HashMap<>(); m.put("foo", "bar"); - w.setCommitData(m); + w.setLiveCommitData(m.entrySet()); w.commit(); - // setCommitData and also commit, with no other changes, should count as an NRT change: + // setLiveCommitData and also commit, with no other changes, should count as an NRT change: DirectoryReader r2 = DirectoryReader.openIfChanged(r); assertNotNull(r2); IOUtils.close(r, r2, w, dir); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterCommit.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterCommit.java index 6148ba129a1..e94621a34c9 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterCommit.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterCommit.java @@ -19,6 +19,7 @@ package org.apache.lucene.index; import java.io.IOException; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; @@ -421,13 +422,13 @@ public class TestIndexWriterCommit extends LuceneTestCase { // commit to "first" Map commitData = new HashMap<>(); commitData.put("tag", "first"); - w.setCommitData(commitData); + w.setLiveCommitData(commitData.entrySet()); w.commit(); // commit to "second" w.addDocument(doc); commitData.put("tag", "second"); - w.setCommitData(commitData); + w.setLiveCommitData(commitData.entrySet()); w.close(); // open "first" with IndexWriter @@ -450,7 +451,7 @@ public class TestIndexWriterCommit extends LuceneTestCase { // commit IndexWriter to "third" w.addDocument(doc); commitData.put("tag", "third"); - w.setCommitData(commitData); + w.setLiveCommitData(commitData.entrySet()); w.close(); // make sure "second" commit is still there @@ -632,7 +633,7 @@ public class TestIndexWriterCommit extends LuceneTestCase { TestIndexWriter.addDoc(w); Map data = new HashMap<>(); data.put("label", "test1"); - w.setCommitData(data); + w.setLiveCommitData(data.entrySet()); w.close(); r = DirectoryReader.open(dir); @@ -663,4 +664,32 @@ public class TestIndexWriterCommit extends LuceneTestCase { r.close(); dir.close(); } + + // LUCENE-7335: make sure commit data is late binding + public void testCommitDataIsLive() throws Exception { + Directory dir = newDirectory(); + IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))); + w.addDocument(new Document()); + + final Map commitData = new HashMap<>(); + commitData.put("foo", "bar"); + + // make sure "foo" / "bar" doesn't take + w.setLiveCommitData(commitData.entrySet()); + + commitData.clear(); + commitData.put("boo", "baz"); + + // this finally does the commit, and should burn "boo" / "baz" + w.close(); + + List commits = DirectoryReader.listCommits(dir); + assertEquals(1, commits.size()); + + IndexCommit commit = commits.get(0); + Map data = commit.getUserData(); + assertEquals(1, data.size()); + assertEquals("baz", data.get("boo")); + dir.close(); + } } diff --git a/lucene/core/src/test/org/apache/lucene/index/TestTransactionRollback.java b/lucene/core/src/test/org/apache/lucene/index/TestTransactionRollback.java index 3abb3922685..e7de028cd68 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestTransactionRollback.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestTransactionRollback.java @@ -71,7 +71,7 @@ public class TestTransactionRollback extends LuceneTestCase { .setIndexCommit(last)); Map data = new HashMap<>(); data.put("index", "Rolled back to 1-"+id); - w.setCommitData(data); + w.setLiveCommitData(data.entrySet()); w.close(); } @@ -142,7 +142,7 @@ public class TestTransactionRollback extends LuceneTestCase { if (currentRecordId%10 == 0) { Map data = new HashMap<>(); data.put("index", "records 1-"+currentRecordId); - w.setCommitData(data); + w.setLiveCommitData(data.entrySet()); w.commit(); } } diff --git a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyReader.java b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyReader.java index 319377a2013..f631d3be4d4 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyReader.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyReader.java @@ -205,7 +205,7 @@ public abstract class TaxonomyReader implements Closeable { /** * Retrieve user committed data. * - * @see TaxonomyWriter#setCommitData(Map) + * @see TaxonomyWriter#setLiveCommitData(Iterable) */ public abstract Map getCommitUserData() throws IOException; diff --git a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyWriter.java b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyWriter.java index 763fe973742..1561e2abe1a 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyWriter.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyWriter.java @@ -20,6 +20,7 @@ import java.io.Closeable; import java.io.IOException; import java.util.Map; +import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.TwoPhaseCommit; /** @@ -105,19 +106,14 @@ public interface TaxonomyWriter extends Closeable, TwoPhaseCommit { public int getSize(); /** - * Sets the commit user data map. That method is considered a transaction and - * will be {@link #commit() committed} even if no other changes were made to - * the writer instance. - *

    - * NOTE: the map is cloned internally, therefore altering the map's - * contents after calling this method has no effect. + * Sets the commit user data iterable. See {@link IndexWriter#setLiveCommitData}. */ - public void setCommitData(Map commitUserData); + public void setLiveCommitData(Iterable> commitUserData); /** - * Returns the commit user data map that was set on - * {@link #setCommitData(Map)}. + * Returns the commit user data iterable that was set on + * {@link #setLiveCommitData(Iterable)}. */ - public Map getCommitData(); + public Iterable> getLiveCommitData(); } diff --git a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyWriter.java b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyWriter.java index 8e0841e4417..632d74bff8f 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyWriter.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyWriter.java @@ -584,31 +584,42 @@ public class DirectoryTaxonomyWriter implements TaxonomyWriter { public synchronized long commit() throws IOException { ensureOpen(); // LUCENE-4972: if we always call setCommitData, we create empty commits - String epochStr = indexWriter.getCommitData().get(INDEX_EPOCH); + + Map data = new HashMap<>(); + Iterable> iter = indexWriter.getLiveCommitData(); + if (iter != null) { + for(Map.Entry ent : iter) { + data.put(ent.getKey(), ent.getValue()); + } + } + + String epochStr = data.get(INDEX_EPOCH); if (epochStr == null || Long.parseLong(epochStr, 16) != indexEpoch) { - indexWriter.setCommitData(combinedCommitData(indexWriter.getCommitData())); + indexWriter.setLiveCommitData(combinedCommitData(indexWriter.getLiveCommitData())); } return indexWriter.commit(); } /** Combine original user data with the taxonomy epoch. */ - private Map combinedCommitData(Map commitData) { + private Iterable> combinedCommitData(Iterable> commitData) { Map m = new HashMap<>(); if (commitData != null) { - m.putAll(commitData); + for(Map.Entry ent : commitData) { + m.put(ent.getKey(), ent.getValue()); + } } m.put(INDEX_EPOCH, Long.toString(indexEpoch, 16)); - return m; + return m.entrySet(); } @Override - public void setCommitData(Map commitUserData) { - indexWriter.setCommitData(combinedCommitData(commitUserData)); + public void setLiveCommitData(Iterable> commitUserData) { + indexWriter.setLiveCommitData(combinedCommitData(commitUserData)); } @Override - public Map getCommitData() { - return combinedCommitData(indexWriter.getCommitData()); + public Iterable> getLiveCommitData() { + return combinedCommitData(indexWriter.getLiveCommitData()); } /** @@ -619,9 +630,16 @@ public class DirectoryTaxonomyWriter implements TaxonomyWriter { public synchronized long prepareCommit() throws IOException { ensureOpen(); // LUCENE-4972: if we always call setCommitData, we create empty commits - String epochStr = indexWriter.getCommitData().get(INDEX_EPOCH); + Map data = new HashMap<>(); + Iterable> iter = indexWriter.getLiveCommitData(); + if (iter != null) { + for(Map.Entry ent : iter) { + data.put(ent.getKey(), ent.getValue()); + } + } + String epochStr = data.get(INDEX_EPOCH); if (epochStr == null || Long.parseLong(epochStr, 16) != indexEpoch) { - indexWriter.setCommitData(combinedCommitData(indexWriter.getCommitData())); + indexWriter.setLiveCommitData(combinedCommitData(indexWriter.getLiveCommitData())); } return indexWriter.prepareCommit(); } diff --git a/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestDirectoryTaxonomyWriter.java b/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestDirectoryTaxonomyWriter.java index 2edee33fd7f..416b823c78d 100644 --- a/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestDirectoryTaxonomyWriter.java +++ b/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestDirectoryTaxonomyWriter.java @@ -94,7 +94,7 @@ public class TestDirectoryTaxonomyWriter extends FacetTestCase { taxoWriter.addCategory(new FacetLabel("b")); Map userCommitData = new HashMap<>(); userCommitData.put("testing", "1 2 3"); - taxoWriter.setCommitData(userCommitData); + taxoWriter.setLiveCommitData(userCommitData.entrySet()); taxoWriter.close(); DirectoryReader r = DirectoryReader.open(dir); assertEquals("2 categories plus root should have been committed to the underlying directory", 3, r.numDocs()); @@ -109,14 +109,22 @@ public class TestDirectoryTaxonomyWriter extends FacetTestCase { // that the taxonomy index has been recreated. taxoWriter = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE_OR_APPEND, NO_OP_CACHE); taxoWriter.addCategory(new FacetLabel("c")); // add a category so that commit will happen - taxoWriter.setCommitData(new HashMap(){{ + taxoWriter.setLiveCommitData(new HashMap(){{ put("just", "data"); - }}); + }}.entrySet()); taxoWriter.commit(); // verify taxoWriter.getCommitData() + Map data = new HashMap<>(); + Iterable> iter = taxoWriter.getLiveCommitData(); + if (iter != null) { + for(Map.Entry ent : iter) { + data.put(ent.getKey(), ent.getValue()); + } + } + assertNotNull(DirectoryTaxonomyWriter.INDEX_EPOCH - + " not found in taoxWriter.commitData", taxoWriter.getCommitData().get(DirectoryTaxonomyWriter.INDEX_EPOCH)); + + " not found in taoxWriter.commitData", data.get(DirectoryTaxonomyWriter.INDEX_EPOCH)); taxoWriter.close(); r = DirectoryReader.open(dir); @@ -170,9 +178,9 @@ public class TestDirectoryTaxonomyWriter extends FacetTestCase { private void touchTaxo(DirectoryTaxonomyWriter taxoWriter, FacetLabel cp) throws IOException { taxoWriter.addCategory(cp); - taxoWriter.setCommitData(new HashMap(){{ + taxoWriter.setLiveCommitData(new HashMap(){{ put("just", "data"); - }}); + }}.entrySet()); taxoWriter.commit(); } diff --git a/lucene/replicator/src/java/org/apache/lucene/replicator/nrt/PrimaryNode.java b/lucene/replicator/src/java/org/apache/lucene/replicator/nrt/PrimaryNode.java index a045cde4cb5..749f54e716f 100644 --- a/lucene/replicator/src/java/org/apache/lucene/replicator/nrt/PrimaryNode.java +++ b/lucene/replicator/src/java/org/apache/lucene/replicator/nrt/PrimaryNode.java @@ -83,10 +83,16 @@ public abstract class PrimaryNode extends Node { message("IWC:\n" + writer.getConfig()); message("dir:\n" + writer.getDirectory()); - message("commitData: " + writer.getCommitData()); + message("commitData: " + writer.getLiveCommitData()); // Record our primaryGen in the userData, and set initial version to 0: - Map commitData = new HashMap<>(writer.getCommitData()); + Map commitData = new HashMap<>(); + Iterable> iter = writer.getLiveCommitData(); + if (iter != null) { + for(Map.Entry ent : iter) { + commitData.put(ent.getKey(), ent.getValue()); + } + } commitData.put(PRIMARY_GEN_KEY, Long.toString(primaryGen)); if (commitData.get(VERSION_KEY) == null) { commitData.put(VERSION_KEY, "0"); @@ -94,7 +100,7 @@ public abstract class PrimaryNode extends Node { } else { message("keep current commitData version=" + commitData.get(VERSION_KEY)); } - writer.setCommitData(commitData, false); + writer.setLiveCommitData(commitData.entrySet(), false); // We forcefully advance the SIS version to an unused future version. This is necessary if the previous primary crashed and we are // starting up on an "older" index, else versions can be illegally reused but show different results: @@ -153,10 +159,16 @@ public abstract class PrimaryNode extends Node { } public synchronized long getLastCommitVersion() { - String s = curInfos.getUserData().get(VERSION_KEY); + Iterable> iter = writer.getLiveCommitData(); + assert iter != null; + for(Map.Entry ent : iter) { + if (ent.getKey().equals(VERSION_KEY)) { + return Long.parseLong(ent.getValue()); + } + } + // In ctor we always install an initial version: - assert s != null; - return Long.parseLong(s); + throw new AssertionError("missing VERSION_KEY"); } @Override @@ -167,7 +179,7 @@ public abstract class PrimaryNode extends Node { // on xlog replay we are replaying more ops than necessary. commitData.put(VERSION_KEY, Long.toString(copyState.version)); message("top: commit commitData=" + commitData); - writer.setCommitData(commitData, false); + writer.setLiveCommitData(commitData.entrySet(), false); writer.commit(); } diff --git a/lucene/replicator/src/test/org/apache/lucene/replicator/IndexAndTaxonomyReplicationClientTest.java b/lucene/replicator/src/test/org/apache/lucene/replicator/IndexAndTaxonomyReplicationClientTest.java index 069e72d02fa..9ccb2c7a776 100644 --- a/lucene/replicator/src/test/org/apache/lucene/replicator/IndexAndTaxonomyReplicationClientTest.java +++ b/lucene/replicator/src/test/org/apache/lucene/replicator/IndexAndTaxonomyReplicationClientTest.java @@ -172,9 +172,9 @@ public class IndexAndTaxonomyReplicationClientTest extends ReplicatorTestCase { private Revision createRevision(final int id) throws IOException { publishIndexWriter.addDocument(newDocument(publishTaxoWriter, id)); - publishIndexWriter.setCommitData(new HashMap() {{ + publishIndexWriter.setLiveCommitData(new HashMap() {{ put(VERSION_ID, Integer.toString(id, 16)); - }}); + }}.entrySet()); publishIndexWriter.commit(); publishTaxoWriter.commit(); return new IndexAndTaxonomyRevision(publishIndexWriter, publishTaxoWriter); diff --git a/lucene/replicator/src/test/org/apache/lucene/replicator/IndexReplicationClientTest.java b/lucene/replicator/src/test/org/apache/lucene/replicator/IndexReplicationClientTest.java index 64f539abba1..3d724921dc7 100644 --- a/lucene/replicator/src/test/org/apache/lucene/replicator/IndexReplicationClientTest.java +++ b/lucene/replicator/src/test/org/apache/lucene/replicator/IndexReplicationClientTest.java @@ -122,9 +122,9 @@ public class IndexReplicationClientTest extends ReplicatorTestCase { private Revision createRevision(final int id) throws IOException { publishWriter.addDocument(new Document()); - publishWriter.setCommitData(new HashMap() {{ + publishWriter.setLiveCommitData(new HashMap() {{ put(VERSION_ID, Integer.toString(id, 16)); - }}); + }}.entrySet()); publishWriter.commit(); return new IndexRevision(publishWriter); } diff --git a/lucene/replicator/src/test/org/apache/lucene/replicator/LocalReplicatorTest.java b/lucene/replicator/src/test/org/apache/lucene/replicator/LocalReplicatorTest.java index d88218e8acf..ce91e2f08dd 100644 --- a/lucene/replicator/src/test/org/apache/lucene/replicator/LocalReplicatorTest.java +++ b/lucene/replicator/src/test/org/apache/lucene/replicator/LocalReplicatorTest.java @@ -65,9 +65,9 @@ public class LocalReplicatorTest extends ReplicatorTestCase { private Revision createRevision(final int id) throws IOException { sourceWriter.addDocument(new Document()); - sourceWriter.setCommitData(new HashMap() {{ + sourceWriter.setLiveCommitData(new HashMap() {{ put(VERSION_ID, Integer.toString(id, 16)); - }}); + }}.entrySet()); sourceWriter.commit(); return new IndexRevision(sourceWriter); } diff --git a/lucene/replicator/src/test/org/apache/lucene/replicator/http/HttpReplicatorTest.java b/lucene/replicator/src/test/org/apache/lucene/replicator/http/HttpReplicatorTest.java index 7e302acdf6b..8dfec5b9337 100644 --- a/lucene/replicator/src/test/org/apache/lucene/replicator/http/HttpReplicatorTest.java +++ b/lucene/replicator/src/test/org/apache/lucene/replicator/http/HttpReplicatorTest.java @@ -93,7 +93,7 @@ public class HttpReplicatorTest extends ReplicatorTestCase { private void publishRevision(int id) throws IOException { Document doc = new Document(); writer.addDocument(doc); - writer.setCommitData(Collections.singletonMap("ID", Integer.toString(id, 16))); + writer.setLiveCommitData(Collections.singletonMap("ID", Integer.toString(id, 16)).entrySet()); writer.commit(); serverReplicator.publish(new IndexRevision(writer)); } diff --git a/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java b/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java index 31eb94c81e7..8c3c74964c2 100644 --- a/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java +++ b/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java @@ -523,7 +523,7 @@ public class DirectUpdateHandler2 extends UpdateHandler implements SolrCoreState final Map commitData = new HashMap<>(); commitData.put(SolrIndexWriter.COMMIT_TIME_MSEC_KEY, String.valueOf(System.currentTimeMillis())); - iw.setCommitData(commitData); + iw.setLiveCommitData(commitData.entrySet()); } public void prepareCommit(CommitUpdateCommand cmd) throws IOException {