diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 89a672ae671..407d71970dd 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -138,6 +138,9 @@ Bug Fixes * SOLR-9494: Use of {!collapse} sometimes doesn't correctly return true for Collector.needsScores(), especially when the query was cached. This can cause an exception when 'q' is a SpanQuery or potentially others. (David Smiley) +* SOLR-9408: Fix TreeMergeOutputFormat to add timestamp metadata to a commit. SolrCloud replication relies on this. + (Jessica Cheng Mallet via Varun Thacker) + ================== 6.2.0 ================== diff --git a/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/TreeMergeOutputFormat.java b/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/TreeMergeOutputFormat.java index 49f844e7674..e3487adad21 100644 --- a/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/TreeMergeOutputFormat.java +++ b/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/TreeMergeOutputFormat.java @@ -25,6 +25,7 @@ import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; +import com.google.common.base.Preconditions; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.io.Text; @@ -40,12 +41,11 @@ import org.apache.lucene.index.TieredMergePolicy; import org.apache.lucene.misc.IndexMergeTool; import org.apache.lucene.store.Directory; import org.apache.solr.store.hdfs.HdfsDirectory; +import org.apache.solr.update.SolrIndexWriter; import org.apache.solr.util.RTimer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.base.Preconditions; - /** * See {@link IndexMergeTool}. */ @@ -151,7 +151,7 @@ public class TreeMergeOutputFormat extends FileOutputFormat LOG.info("Optimizing Solr: forcing tree merge down to {} segments", maxSegments); timer = new RTimer(); if (maxSegments < Integer.MAX_VALUE) { - writer.forceMerge(maxSegments); + writer.forceMerge(maxSegments); // TODO: consider perf enhancement for no-deletes merges: bulk-copy the postings data // see http://lucene.472066.n3.nabble.com/Experience-with-large-merge-factors-tp1637832p1647046.html } @@ -161,6 +161,10 @@ public class TreeMergeOutputFormat extends FileOutputFormat } LOG.info("Optimizing Solr: done forcing tree merge down to {} segments in {}ms", maxSegments, timer.getTime()); + // Set Solr's commit data so the created index is usable by SolrCloud. E.g. Currently SolrCloud relies on + // commitTimeMSec in the commit data to do replication. + SolrIndexWriter.setCommitData(writer); + timer = new RTimer(); LOG.info("Optimizing Solr: Closing index writer"); writer.close(); diff --git a/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java b/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java index 12552cd5dad..258408dc56e 100644 --- a/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java +++ b/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java @@ -20,10 +20,8 @@ import java.io.IOException; import java.lang.invoke.MethodHandles; import java.net.URL; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Locale; -import java.util.Map; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; import java.util.concurrent.atomic.LongAdder; @@ -47,7 +45,6 @@ import org.apache.solr.common.SolrException; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.SimpleOrderedMap; -import org.apache.solr.common.util.SuppressForbidden; import org.apache.solr.core.SolrConfig.UpdateHandlerInfo; import org.apache.solr.core.SolrCore; import org.apache.solr.request.LocalSolrQueryRequest; @@ -516,19 +513,7 @@ public class DirectUpdateHandler2 extends UpdateHandler implements SolrCoreState return rc; } - - @SuppressForbidden(reason = "Need currentTimeMillis, commit time should be used only for debugging purposes, " + - " but currently suspiciously used for replication as well") - /* - Also see SolrIndexSplitter.setCommitData - */ - private void setCommitData(IndexWriter iw) { - final Map commitData = new HashMap<>(); - commitData.put(SolrIndexWriter.COMMIT_TIME_MSEC_KEY, - String.valueOf(System.currentTimeMillis())); - iw.setLiveCommitData(commitData.entrySet()); - } - + public void prepareCommit(CommitUpdateCommand cmd) throws IOException { boolean error=true; @@ -537,7 +522,7 @@ public class DirectUpdateHandler2 extends UpdateHandler implements SolrCoreState log.info("start "+cmd); RefCounted iw = solrCoreState.getIndexWriter(core); try { - setCommitData(iw.get()); + SolrIndexWriter.setCommitData(iw.get()); iw.get().prepareCommit(); } finally { iw.decref(); @@ -618,7 +603,7 @@ public class DirectUpdateHandler2 extends UpdateHandler implements SolrCoreState // SolrCore.verbose("writer.commit() start writer=",writer); if (writer.hasUncommittedChanges()) { - setCommitData(writer); + SolrIndexWriter.setCommitData(writer); writer.commit(); } else { log.info("No uncommitted changes. Skipping IW.commit."); @@ -803,7 +788,7 @@ public class DirectUpdateHandler2 extends UpdateHandler implements SolrCoreState } // todo: refactor this shared code (or figure out why a real CommitUpdateCommand can't be used) - setCommitData(writer); + SolrIndexWriter.setCommitData(writer); writer.commit(); synchronized (solrCoreState.getUpdateLock()) { diff --git a/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java b/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java index 14e70632d86..bd4612e5cc4 100644 --- a/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java +++ b/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java @@ -19,17 +19,15 @@ package org.apache.solr.update; import java.io.IOException; import java.lang.invoke.MethodHandles; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; import org.apache.lucene.index.CodecReader; +import org.apache.lucene.index.Fields; import org.apache.lucene.index.FilterCodecReader; +import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.PostingsEnum; -import org.apache.lucene.index.Fields; -import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.SlowCodecReaderWrapper; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; @@ -42,7 +40,6 @@ import org.apache.lucene.util.IOUtils; import org.apache.solr.common.cloud.CompositeIdRouter; import org.apache.solr.common.cloud.DocRouter; import org.apache.solr.common.cloud.HashBasedRouter; -import org.apache.solr.common.util.SuppressForbidden; import org.apache.solr.core.SolrCore; import org.apache.solr.schema.SchemaField; import org.apache.solr.search.BitsFilteredPostingsEnum; @@ -140,7 +137,7 @@ public class SolrIndexSplitter { // we commit explicitly instead of sending a CommitUpdateCommand through the processor chain // because the sub-shard cores will just ignore such a commit because the update log is not // in active state at this time. - setCommitData(iw); + SolrIndexWriter.setCommitData(iw); iw.commit(); success = true; } finally { @@ -159,14 +156,6 @@ public class SolrIndexSplitter { } - @SuppressForbidden(reason = "Need currentTimeMillis, commit time should be used only for debugging purposes, " + - " but currently suspiciously used for replication as well") - private void setCommitData(IndexWriter iw) { - final Map commitData = new HashMap<>(); - commitData.put(SolrIndexWriter.COMMIT_TIME_MSEC_KEY, String.valueOf(System.currentTimeMillis())); - iw.setLiveCommitData(commitData.entrySet()); - } - FixedBitSet[] split(LeafReaderContext readerContext) throws IOException { LeafReader reader = readerContext.reader(); FixedBitSet[] docSets = new FixedBitSet[numPieces]; diff --git a/solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java b/solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java index 8599da1ed3c..fcfc559475c 100644 --- a/solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java +++ b/solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java @@ -18,6 +18,8 @@ package org.apache.solr.update; import java.io.IOException; import java.lang.invoke.MethodHandles; +import java.util.HashMap; +import java.util.Map; import java.util.concurrent.atomic.AtomicLong; import org.apache.lucene.codecs.Codec; @@ -27,8 +29,9 @@ import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.store.Directory; import org.apache.lucene.util.InfoStream; import org.apache.solr.common.util.IOUtils; -import org.apache.solr.core.DirectoryFactory.DirContext; +import org.apache.solr.common.util.SuppressForbidden; import org.apache.solr.core.DirectoryFactory; +import org.apache.solr.core.DirectoryFactory.DirContext; import org.apache.solr.core.SolrCore; import org.apache.solr.schema.IndexSchema; import org.slf4j.Logger; @@ -86,7 +89,16 @@ public class SolrIndexWriter extends IndexWriter { this.directory = directory; numOpens.incrementAndGet(); } - + + @SuppressForbidden(reason = "Need currentTimeMillis, commit time should be used only for debugging purposes, " + + " but currently suspiciously used for replication as well") + public static void setCommitData(IndexWriter iw) { + log.info("Calling setCommitData with IW:" + iw.toString()); + final Map commitData = new HashMap<>(); + commitData.put(COMMIT_TIME_MSEC_KEY, String.valueOf(System.currentTimeMillis())); + iw.setLiveCommitData(commitData.entrySet()); + } + private void setDirectoryFactory(DirectoryFactory factory) { this.directoryFactory = factory; }