From 532f443146020845944d65f5460daa23b77d86c6 Mon Sep 17 00:00:00 2001 From: Shalin Shekhar Mangar Date: Mon, 20 Jun 2011 07:32:12 +0000 Subject: [PATCH] SOLR-1331 -- Added a srcCore parameter to CoreAdminHandler's mergeindexes action to merge one or more cores' indexes to a target core git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1137533 13f79535-47bb-0310-9956-ffa450edef68 --- solr/CHANGES.txt | 3 + .../solr/common/params/CoreAdminParams.java | 4 ++ .../solr/handler/admin/CoreAdminHandler.java | 71 +++++++++++++++---- .../solr/update/DirectUpdateHandler2.java | 7 +- .../solr/update/MergeIndexesCommand.java | 19 +++-- .../solrj/request/CoreAdminRequest.java | 17 ++++- .../solrj/MergeIndexesExampleTestBase.java | 32 +++++++-- 7 files changed, 120 insertions(+), 33 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 3bbe0fbb4b4..be009d6b7b9 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -266,6 +266,9 @@ New Features Karsten Sperling, Michael Gundlach, Oleg Gnatovskiy, Thomas Traeger, Harish Agarwal, yonik, Michael McCandless, Bill Bell) +* SOLR-1331 -- Added a srcCore parameter to CoreAdminHandler's mergeindexes action + to merge one or more cores' indexes to a target core (shalin) + Optimizations ---------------------- diff --git a/solr/src/common/org/apache/solr/common/params/CoreAdminParams.java b/solr/src/common/org/apache/solr/common/params/CoreAdminParams.java index 6171262df07..0691264e240 100644 --- a/solr/src/common/org/apache/solr/common/params/CoreAdminParams.java +++ b/solr/src/common/org/apache/solr/common/params/CoreAdminParams.java @@ -59,6 +59,10 @@ public interface CoreAdminParams * The directories are specified by multiple indexDir parameters. */ public final static String INDEX_DIR = "indexDir"; + /** If you merge indexes, what is the source core's name + * More than one source core can be specified by multiple srcCore parameters */ + public final static String SRC_CORE = "srcCore"; + /** The collection name in solr cloud */ public final static String COLLECTION = "collection"; diff --git a/solr/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java b/solr/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java index 39b9a048e22..f004c0ce888 100644 --- a/solr/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java +++ b/solr/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java @@ -17,6 +17,9 @@ package org.apache.solr.handler.admin; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.util.IOUtils; import org.apache.solr.cloud.CloudDescriptor; import org.apache.solr.common.SolrException; import org.apache.solr.common.params.CoreAdminParams; @@ -42,7 +45,9 @@ import org.apache.lucene.store.Directory; import java.io.File; import java.io.IOException; +import java.util.ArrayList; import java.util.Date; +import java.util.List; /** * @@ -171,22 +176,53 @@ public class CoreAdminHandler extends RequestHandlerBase { } protected boolean handleMergeAction(SolrQueryRequest req, SolrQueryResponse rsp) throws IOException { - boolean doPersist = false; SolrParams params = req.getParams(); - SolrParams required = params.required(); - String cname = required.get(CoreAdminParams.CORE); + String cname = params.required().get(CoreAdminParams.CORE); SolrCore core = coreContainer.getCore(cname); SolrQueryRequest wrappedReq = null; + + SolrCore[] sourceCores = null; + RefCounted[] searchers = null; + // stores readers created from indexDir param values + IndexReader[] readersToBeClosed = null; if (core != null) { try { - doPersist = coreContainer.isPersistent(); + String[] dirNames = params.getParams(CoreAdminParams.INDEX_DIR); + if (dirNames == null || dirNames.length == 0) { + String[] sources = params.getParams("srcCore"); + if (sources == null || sources.length == 0) + throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, + "At least one indexDir or srcCore must be specified"); - String[] dirNames = required.getParams(CoreAdminParams.INDEX_DIR); + sourceCores = new SolrCore[sources.length]; + for (int i = 0; i < sources.length; i++) { + String source = sources[i]; + SolrCore srcCore = coreContainer.getCore(source); + if (srcCore == null) + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, + "Core: " + source + " does not exist"); + sourceCores[i] = srcCore; + } + } else { + readersToBeClosed = new IndexReader[dirNames.length]; + DirectoryFactory dirFactory = core.getDirectoryFactory(); + for (int i = 0; i < dirNames.length; i++) { + readersToBeClosed[i] = IndexReader.open(dirFactory.open(dirNames[i]), true); + } + } - DirectoryFactory dirFactory = core.getDirectoryFactory(); - Directory[] dirs = new Directory[dirNames.length]; - for (int i = 0; i < dirNames.length; i++) { - dirs[i] = dirFactory.open(dirNames[i]); + IndexReader[] readers = null; + if (readersToBeClosed != null) { + readers = readersToBeClosed; + } else { + readers = new IndexReader[sourceCores.length]; + searchers = new RefCounted[sourceCores.length]; + for (int i = 0; i < sourceCores.length; i++) { + SolrCore solrCore = sourceCores[i]; + // record the searchers so that we can decref + searchers[i] = solrCore.getSearcher(); + readers[i] = searchers[i].get().getIndexReader(); + } } UpdateRequestProcessorChain processorChain = @@ -194,13 +230,24 @@ public class CoreAdminHandler extends RequestHandlerBase { wrappedReq = new LocalSolrQueryRequest(core, req.getParams()); UpdateRequestProcessor processor = processorChain.createProcessor(wrappedReq, rsp); - processor.processMergeIndexes(new MergeIndexesCommand(dirs, req)); + processor.processMergeIndexes(new MergeIndexesCommand(readers, req)); } finally { + if (searchers != null) { + for (RefCounted searcher : searchers) { + if (searcher != null) searcher.decref(); + } + } + if (sourceCores != null) { + for (SolrCore solrCore : sourceCores) { + if (solrCore != null) solrCore.close(); + } + } + if (readersToBeClosed != null) IOUtils.closeSafely(true, readersToBeClosed); + if (wrappedReq != null) wrappedReq.close(); core.close(); - wrappedReq.close(); } } - return doPersist; + return coreContainer.isPersistent(); } /** diff --git a/solr/src/java/org/apache/solr/update/DirectUpdateHandler2.java b/solr/src/java/org/apache/solr/update/DirectUpdateHandler2.java index 5106b1cb0aa..69a1454a167 100644 --- a/solr/src/java/org/apache/solr/update/DirectUpdateHandler2.java +++ b/solr/src/java/org/apache/solr/update/DirectUpdateHandler2.java @@ -20,6 +20,7 @@ package org.apache.solr.update; +import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.Term; import org.apache.lucene.queryParser.ParseException; @@ -277,10 +278,10 @@ public class DirectUpdateHandler2 extends UpdateHandler { try { log.info("start " + cmd); - Directory[] dirs = cmd.dirs; - if (dirs != null && dirs.length > 0) { + IndexReader[] readers = cmd.readers; + if (readers != null && readers.length > 0) { openWriter(); - writer.addIndexes(dirs); + writer.addIndexes(readers); rc = 1; } else { rc = 0; diff --git a/solr/src/java/org/apache/solr/update/MergeIndexesCommand.java b/solr/src/java/org/apache/solr/update/MergeIndexesCommand.java index 412d2887083..1226012f03f 100644 --- a/solr/src/java/org/apache/solr/update/MergeIndexesCommand.java +++ b/solr/src/java/org/apache/solr/update/MergeIndexesCommand.java @@ -17,6 +17,7 @@ package org.apache.solr.update; +import org.apache.lucene.index.IndexReader; import org.apache.lucene.store.Directory; import org.apache.solr.request.SolrQueryRequest; @@ -27,25 +28,21 @@ import org.apache.solr.request.SolrQueryRequest; * */ public class MergeIndexesCommand extends UpdateCommand { - public Directory[] dirs; + public IndexReader[] readers; - public MergeIndexesCommand(SolrQueryRequest req) { - this(null, req); - } - - public MergeIndexesCommand(Directory[] dirs, SolrQueryRequest req) { + public MergeIndexesCommand(IndexReader[] readers, SolrQueryRequest req) { super("mergeIndexes", req); - this.dirs = dirs; + this.readers = readers; } @Override public String toString() { StringBuilder sb = new StringBuilder(commandName); sb.append(':'); - if (dirs != null && dirs.length > 0) { - sb.append(dirs[0]); - for (int i = 1; i < dirs.length; i++) { - sb.append(",").append(dirs[i]); + if (readers != null && readers.length > 0) { + sb.append(readers[0].directory()); + for (int i = 1; i < readers.length; i++) { + sb.append(",").append(readers[i].directory()); } } return sb.toString(); diff --git a/solr/src/solrj/org/apache/solr/client/solrj/request/CoreAdminRequest.java b/solr/src/solrj/org/apache/solr/client/solrj/request/CoreAdminRequest.java index 2e8bfafbdf1..05b4e37f0f1 100644 --- a/solr/src/solrj/org/apache/solr/client/solrj/request/CoreAdminRequest.java +++ b/solr/src/solrj/org/apache/solr/client/solrj/request/CoreAdminRequest.java @@ -119,6 +119,7 @@ public class CoreAdminRequest extends SolrRequest public static class MergeIndexes extends CoreAdminRequest { protected List indexDirs; + protected List srcCores; public MergeIndexes() { action = CoreAdminAction.MERGEINDEXES; @@ -132,6 +133,14 @@ public class CoreAdminRequest extends SolrRequest return indexDirs; } + public List getSrcCores() { + return srcCores; + } + + public void setSrcCores(List srcCores) { + this.srcCores = srcCores; + } + @Override public SolrParams getParams() { if (action == null) { @@ -145,6 +154,11 @@ public class CoreAdminRequest extends SolrRequest params.set(CoreAdminParams.INDEX_DIR, indexDir); } } + if (srcCores != null) { + for (String srcCore : srcCores) { + params.set(CoreAdminParams.SRC_CORE, srcCore); + } + } return params; } } @@ -289,11 +303,12 @@ public class CoreAdminRequest extends SolrRequest } public static CoreAdminResponse mergeIndexes(String name, - String[] indexDirs, SolrServer server) throws SolrServerException, + String[] indexDirs, String[] srcCores, SolrServer server) throws SolrServerException, IOException { CoreAdminRequest.MergeIndexes req = new CoreAdminRequest.MergeIndexes(); req.setCoreName(name); req.setIndexDirs(Arrays.asList(indexDirs)); + req.setSrcCores(Arrays.asList(srcCores)); return req.process(server); } } diff --git a/solr/src/test/org/apache/solr/client/solrj/MergeIndexesExampleTestBase.java b/solr/src/test/org/apache/solr/client/solrj/MergeIndexesExampleTestBase.java index b3361030009..95f408057ef 100644 --- a/solr/src/test/org/apache/solr/client/solrj/MergeIndexesExampleTestBase.java +++ b/solr/src/test/org/apache/solr/client/solrj/MergeIndexesExampleTestBase.java @@ -17,6 +17,7 @@ package org.apache.solr.client.solrj; +import org.apache.solr.client.solrj.request.AbstractUpdateRequest; import org.apache.solr.client.solrj.request.CoreAdminRequest; import org.apache.solr.client.solrj.request.QueryRequest; import org.apache.solr.client.solrj.request.UpdateRequest; @@ -26,6 +27,8 @@ import org.apache.solr.core.CoreContainer; import org.apache.solr.core.SolrCore; import org.apache.solr.util.ExternalPaths; +import java.io.IOException; + /** * Abstract base class for testing merge indexes command * @@ -79,9 +82,9 @@ public abstract class MergeIndexesExampleTestBase extends SolrExampleTestBase { protected abstract String getIndexDirCore1(); - public void testMergeIndexes() throws Exception { + private UpdateRequest setupCores() throws SolrServerException, IOException { UpdateRequest up = new UpdateRequest(); - up.setAction(ACTION.COMMIT, true, true); + up.setAction(AbstractUpdateRequest.ACTION.COMMIT, true, true); up.deleteByQuery("*:*"); up.process(getSolrCore0()); up.process(getSolrCore1()); @@ -119,11 +122,28 @@ public abstract class MergeIndexesExampleTestBase extends SolrExampleTestBase { assertEquals(1, getSolrCore1().query(new SolrQuery("id:BBB")).getResults().size()); + return up; + } + + public void testMergeIndexesByDirName() throws Exception { + UpdateRequest up = setupCores(); + // Now get the index directory of core1 and merge with core0 - String indexDir = getIndexDirCore1(); - String name = "core0"; - SolrServer coreadmin = getSolrAdmin(); - CoreAdminRequest.mergeIndexes(name, new String[] { indexDir }, coreadmin); + CoreAdminRequest.mergeIndexes("core0", new String[] {getIndexDirCore1()}, new String[0], getSolrAdmin()); + + // Now commit the merged index + up.clear(); // just do commit + up.process(getSolrCore0()); + + assertEquals(1, + getSolrCore0().query(new SolrQuery("id:AAA")).getResults().size()); + assertEquals(1, + getSolrCore0().query(new SolrQuery("id:BBB")).getResults().size()); + } + + public void testMergeIndexesByCoreName() throws Exception { + UpdateRequest up = setupCores(); + CoreAdminRequest.mergeIndexes("core0", new String[0], new String[] {"core1"}, getSolrAdmin()); // Now commit the merged index up.clear(); // just do commit