From d37c98d95a16a05dde1cb9d69b7682aa8eac95e5 Mon Sep 17 00:00:00 2001 From: Shalin Shekhar Mangar Date: Sat, 9 Feb 2013 17:19:19 +0000 Subject: [PATCH] SOLR-3755: Test for SolrIndexSplitter git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1444397 13f79535-47bb-0310-9956-ffa450edef68 --- .../solr/update/SolrIndexSplitterTest.java | 176 ++++++++++++++++++ 1 file changed, 176 insertions(+) create mode 100644 solr/core/src/test/org/apache/solr/update/SolrIndexSplitterTest.java diff --git a/solr/core/src/test/org/apache/solr/update/SolrIndexSplitterTest.java b/solr/core/src/test/org/apache/solr/update/SolrIndexSplitterTest.java new file mode 100644 index 00000000000..8731f67ef8c --- /dev/null +++ b/solr/core/src/test/org/apache/solr/update/SolrIndexSplitterTest.java @@ -0,0 +1,176 @@ +package org.apache.solr.update; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import com.google.common.collect.Lists; +import org.apache.commons.io.FileUtils; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.Term; +import org.apache.lucene.store.Directory; +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer; +import org.apache.solr.common.cloud.DocRouter; +import org.apache.solr.common.cloud.PlainIdRouter; +import org.apache.solr.common.util.Hash; +import org.apache.solr.core.CoreDescriptor; +import org.apache.solr.core.DirectoryFactory; +import org.apache.solr.core.SolrCore; +import org.apache.solr.request.LocalSolrQueryRequest; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.File; +import java.io.UnsupportedEncodingException; +import java.util.List; + +public class SolrIndexSplitterTest extends SolrTestCaseJ4 { + File indexDir1 = null, indexDir2 = null; + + @BeforeClass + public static void beforeClass() throws Exception { + initCore("solrconfig.xml", "schema12.xml"); + } + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + clearIndex(); + assertU(commit()); + indexDir1 = new File(TEMP_DIR, this.getClass().getName() + + "_testSplit1"); + indexDir2 = new File(TEMP_DIR, this.getClass().getName() + + "_testSplit2"); + + if (indexDir1.exists()) { + FileUtils.deleteDirectory(indexDir1); + } + assertTrue("Failed to mkdirs indexDir1 for split index", indexDir1.mkdirs()); + + if (indexDir2.exists()) { + FileUtils.deleteDirectory(indexDir2); + } + assertTrue("Failed to mkdirs indexDir2 for split index", indexDir2.mkdirs()); + } + + @Test + public void testSplitByPaths() throws Exception { + LocalSolrQueryRequest request = null; + try { + // add two docs + String id1 = "dorothy"; + assertU(adoc("id", id1)); + String id2 = "kansas"; + assertU(adoc("id", id2)); + assertU(commit()); + assertJQ(req("q", "*:*"), "/response/numFound==2"); + + // find minHash/maxHash hash ranges + List ranges = getRanges(id1, id2); + + request = lrf.makeRequest("q", "dummy"); + + SplitIndexCommand command = new SplitIndexCommand(request, + Lists.newArrayList(indexDir1.getAbsolutePath(), indexDir2.getAbsolutePath()), null, ranges); + new SolrIndexSplitter(command).split(); + + Directory directory = h.getCore().getDirectoryFactory().get(indexDir1.getAbsolutePath(), + DirectoryFactory.DirContext.DEFAULT, h.getCore().getSolrConfig().indexConfig.lockType); + DirectoryReader reader = DirectoryReader.open(directory); + assertEquals("id:dorothy should be present in split index1", 1, reader.docFreq(new Term("id", "dorothy"))); + assertEquals("id:kansas should not be present in split index1", 0, reader.docFreq(new Term("id", "kansas"))); + assertEquals("split index1 should have only one document", 1, reader.numDocs()); + reader.close(); + h.getCore().getDirectoryFactory().release(directory); + directory = h.getCore().getDirectoryFactory().get(indexDir2.getAbsolutePath(), + DirectoryFactory.DirContext.DEFAULT, h.getCore().getSolrConfig().indexConfig.lockType); + reader = DirectoryReader.open(directory); + assertEquals("id:dorothy should not be present in split index2", 0, reader.docFreq(new Term("id", "dorothy"))); + assertEquals("id:kansas should be present in split index2", 1, reader.docFreq(new Term("id", "kansas"))); + assertEquals("split index2 should have only one document", 1, reader.numDocs()); + reader.close(); + h.getCore().getDirectoryFactory().release(directory); + } finally { + if (request != null) request.close(); // decrefs the searcher + } + } + + @Test + public void testSplitByCores() throws Exception { + // add two docs + String id1 = "dorothy"; + assertU(adoc("id", id1)); + String id2 = "kansas"; + assertU(adoc("id", id2)); + assertU(commit()); + assertJQ(req("q", "*:*"), "/response/numFound==2"); + List ranges = getRanges(id1, id2); + + SolrCore core1 = null, core2 = null; + try { + CoreDescriptor dcore1 = new CoreDescriptor(h.getCoreContainer(), "split1", h.getCore().getCoreDescriptor().getInstanceDir()); + dcore1.setDataDir(indexDir1.getAbsolutePath()); + dcore1.setSchemaName("schema12.xml"); + core1 = h.getCoreContainer().create(dcore1); + h.getCoreContainer().register(core1, false); + + CoreDescriptor dcore2 = new CoreDescriptor(h.getCoreContainer(), "split2", h.getCore().getCoreDescriptor().getInstanceDir()); + dcore2.setDataDir(indexDir2.getAbsolutePath()); + dcore2.setSchemaName("schema12.xml"); + core2 = h.getCoreContainer().create(dcore2); + h.getCoreContainer().register(core2, false); + + LocalSolrQueryRequest request = null; + try { + request = lrf.makeRequest("q", "dummy"); + + SplitIndexCommand command = new SplitIndexCommand(request, null, Lists.newArrayList(core1, core2), ranges); + new SolrIndexSplitter(command).split(); + } finally { + if (request != null) request.close(); + } + EmbeddedSolrServer server1 = new EmbeddedSolrServer(h.getCoreContainer(), "split1"); + EmbeddedSolrServer server2 = new EmbeddedSolrServer(h.getCoreContainer(), "split2"); + server1.commit(true, true); + server2.commit(true, true); + assertEquals("id:dorothy should be present in split index1", 1, server1.query(new SolrQuery("id:dorothy")).getResults().getNumFound()); + assertEquals("id:kansas should not be present in split index1", 0, server1.query(new SolrQuery("id:kansas")).getResults().getNumFound()); + assertEquals("id:dorothy should not be present in split index2", 0, server2.query(new SolrQuery("id:dorothy")).getResults().getNumFound()); + assertEquals("id:kansas should be present in split index2", 1, server2.query(new SolrQuery("id:kansas")).getResults().getNumFound()); + } finally { + h.getCoreContainer().remove("split2"); + h.getCoreContainer().remove("split1"); + if (core2 != null) core2.close(); + if (core1 != null) core1.close(); + } + } + + private List getRanges(String id1, String id2) throws UnsupportedEncodingException { + // find minHash/maxHash hash ranges + byte[] bytes = id1.getBytes("UTF-8"); + int minHash = Hash.murmurhash3_x86_32(bytes, 0, bytes.length, 0); + bytes = id2.getBytes("UTF-8"); + int maxHash = Hash.murmurhash3_x86_32(bytes, 0, bytes.length, 0); + + PlainIdRouter router = new PlainIdRouter(); + DocRouter.Range fullRange = new DocRouter.Range(minHash, maxHash); + return router.partitionRange(2, fullRange); + } +}