SOLR-2002: improve TestReplicationHandler

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@965327 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Robert Muir 2010-07-18 23:58:48 +00:00
parent 828d00a3c7
commit 1f10f8ba7f
1 changed files with 99 additions and 81 deletions

View File

@ -20,7 +20,9 @@ import org.apache.commons.io.IOUtils;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.SimpleFSDirectory;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.TestDistributedSearch;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrServerException;
@ -34,33 +36,36 @@ import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.util.AbstractSolrTestCase;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import static org.junit.Assert.*;
import java.io.*;
import java.net.URL;
import junit.framework.TestCase;
/**
* Test for ReplicationHandler
*
* @version $Id$
* @since 1.4
*/
public class TestReplicationHandler extends TestCase {
public class TestReplicationHandler extends SolrTestCaseJ4 {
private static final String CONF_DIR = "." + File.separator + "solr" + File.separator + "conf" + File.separator;
private static final String SLAVE_CONFIG = CONF_DIR + "solrconfig-slave.xml";
JettySolrRunner masterJetty, slaveJetty;
SolrServer masterClient, slaveClient;
SolrInstance master = null, slave = null;
static JettySolrRunner masterJetty, slaveJetty;
static SolrServer masterClient, slaveClient;
static SolrInstance master = null, slave = null;
String context = "/solr";
static String context = "/solr";
public void setUp() throws Exception {
super.setUp();
@BeforeClass
public static void beforeClass() throws Exception {
master = new SolrInstance("master", null);
master.setUp();
masterJetty = createJetty(master);
@ -72,16 +77,26 @@ public class TestReplicationHandler extends TestCase {
slaveClient = createNewSolrServer(slaveJetty.getLocalPort());
}
@Override
public void tearDown() throws Exception {
@Before
public void setUp() throws Exception {
super.setUp();
masterClient.deleteByQuery("*:*");
masterClient.commit();
rQuery(0, "*:*", masterClient);
slaveClient.deleteByQuery("*:*");
slaveClient.commit();
rQuery(0, "*:*", slaveClient);
}
@AfterClass
public static void afterClass() throws Exception {
masterJetty.stop();
slaveJetty.stop();
master.tearDown();
slave.tearDown();
super.tearDown();
}
private JettySolrRunner createJetty(SolrInstance instance) throws Exception {
private static JettySolrRunner createJetty(SolrInstance instance) throws Exception {
System.setProperty("solr.solr.home", instance.getHomeDir());
System.setProperty("solr.data.dir", instance.getDataDir());
@ -91,7 +106,7 @@ public class TestReplicationHandler extends TestCase {
return jetty;
}
protected SolrServer createNewSolrServer(int port) {
private static SolrServer createNewSolrServer(int port) {
try {
// setup the server...
String url = "http://localhost:" + port + context;
@ -126,6 +141,21 @@ public class TestReplicationHandler extends TestCase {
return res;
}
/** will sleep up to 30 seconds, looking for expectedDocCount */
private NamedList rQuery(int expectedDocCount, String query, SolrServer server) throws Exception {
int timeSlept = 0;
NamedList res = null;
SolrDocumentList docList = null;
do {
res = query(query, server);
docList = (SolrDocumentList) res.get("response");
timeSlept += 100;
Thread.sleep(100);
} while(docList.getNumFound() != expectedDocCount && timeSlept < 30000);
return res;
}
@Test
public void testIndexAndConfigReplication() throws Exception {
//add 500 docs to master
@ -134,24 +164,13 @@ public class TestReplicationHandler extends TestCase {
masterClient.commit();
NamedList masterQueryRsp = query("*:*", masterClient);
NamedList masterQueryRsp = rQuery(500, "*:*", masterClient);
SolrDocumentList masterQueryResult = (SolrDocumentList) masterQueryRsp.get("response");
assertEquals(500, masterQueryResult.getNumFound());
//sleep for pollinterval time 3s, to let slave pull data.
Thread.sleep(3000);
//get docs from slave and check if number is equal to master
NamedList slaveQueryRsp = query("*:*", slaveClient);
NamedList slaveQueryRsp = rQuery(500, "*:*", slaveClient);
SolrDocumentList slaveQueryResult = (SolrDocumentList) slaveQueryRsp.get("response");
if (slaveQueryResult.getNumFound() == 0) {
//try sleeping again in case of slower comp
Thread.sleep(5000);
slaveQueryRsp = query("*:*", slaveClient);
slaveQueryResult = (SolrDocumentList) slaveQueryRsp.get("response");
}
assertEquals(500, slaveQueryResult.getNumFound());
//compare results
@ -180,15 +199,17 @@ public class TestReplicationHandler extends TestCase {
index(masterClient, "id", "2000", "name", "name = " + 2000, "newname", "newname = " + 2000);
masterClient.commit();
//sleep for 3s for replication to happen.
Thread.sleep(3000);
NamedList masterQueryRsp2 = rQuery(1, "*:*", masterClient);
SolrDocumentList masterQueryResult2 = (SolrDocumentList) masterQueryRsp2.get("response");
assertEquals(1, masterQueryResult2.getNumFound());
slaveQueryRsp = query("*:*", slaveClient);
slaveQueryRsp = rQuery(1, "*:*", slaveClient);
SolrDocument d = ((SolrDocumentList) slaveQueryRsp.get("response")).get(0);
assertEquals("newname = 2000", (String) d.getFieldValue("newname"));
}
@Test
public void testIndexAndConfigAliasReplication() throws Exception {
//add 500 docs to master
@ -197,24 +218,14 @@ public class TestReplicationHandler extends TestCase {
masterClient.commit();
NamedList masterQueryRsp = query("*:*", masterClient);
NamedList masterQueryRsp = rQuery(500, "*:*", masterClient);
SolrDocumentList masterQueryResult = (SolrDocumentList) masterQueryRsp.get("response");
assertEquals(500, masterQueryResult.getNumFound());
//sleep for pollinterval time 3s, to let slave pull data.
Thread.sleep(3000);
//get docs from slave and check if number is equal to master
NamedList slaveQueryRsp = query("*:*", slaveClient);
NamedList slaveQueryRsp = rQuery(500, "*:*", slaveClient);
SolrDocumentList slaveQueryResult = (SolrDocumentList) slaveQueryRsp.get("response");
if (slaveQueryResult.getNumFound() == 0) {
//try sleeping again in case of slower comp
Thread.sleep(5000);
slaveQueryRsp = query("*:*", slaveClient);
slaveQueryResult = (SolrDocumentList) slaveQueryRsp.get("response");
}
assertEquals(500, slaveQueryResult.getNumFound());
//compare results
@ -249,20 +260,25 @@ public class TestReplicationHandler extends TestCase {
//add a doc with new field and commit on master to trigger snappull from slave.
index(masterClient, "id", "2000", "name", "name = " + 2000, "newname", "newname = " + 2000);
masterClient.commit();
//sleep for 3s for replication to happen.
Thread.sleep(3000);
NamedList masterQueryRsp2 = rQuery(1, "*:*", masterClient);
SolrDocumentList masterQueryResult2 = (SolrDocumentList) masterQueryRsp2.get("response");
assertEquals(1, masterQueryResult2.getNumFound());
NamedList slaveQueryRsp2 = rQuery(1, "*:*", slaveClient);
SolrDocumentList slaveQueryResult2 = (SolrDocumentList) slaveQueryRsp2.get("response");
assertEquals(1, slaveQueryResult2.getNumFound());
index(slaveClient, "id", "2000", "name", "name = " + 2001, "newname", "newname = " + 2001);
slaveClient.commit();
slaveQueryRsp = query("*:*", slaveClient);
slaveQueryRsp = rQuery(1, "*:*", slaveClient);
SolrDocument d = ((SolrDocumentList) slaveQueryRsp.get("response")).get(0);
assertEquals("newname = 2001", (String) d.getFieldValue("newname"));
}
@Test
public void testStopPoll() throws Exception {
// Test:
// setup master/slave.
@ -274,14 +290,12 @@ public class TestReplicationHandler extends TestCase {
masterClient.commit();
NamedList masterQueryRsp = query("*:*", masterClient);
NamedList masterQueryRsp = rQuery(500, "*:*", masterClient);
SolrDocumentList masterQueryResult = (SolrDocumentList) masterQueryRsp.get("response");
assertEquals(500, masterQueryResult.getNumFound());
//sleep for pollinterval time 3s, to let slave pull data.
Thread.sleep(3000);
//get docs from slave and check if number is equal to master
NamedList slaveQueryRsp = query("*:*", slaveClient);
NamedList slaveQueryRsp = rQuery(500, "*:*", slaveClient);
SolrDocumentList slaveQueryResult = (SolrDocumentList) slaveQueryRsp.get("response");
assertEquals(500, slaveQueryResult.getNumFound());
@ -300,18 +314,24 @@ public class TestReplicationHandler extends TestCase {
}
index(masterClient, "id", 501, "name", "name = " + 501);
masterClient.commit();
//sleep for pollinterval time 3s, to let slave pull data.
//get docs from master and check if number is equal to master
masterQueryRsp = rQuery(501, "*:*", masterClient);
masterQueryResult = (SolrDocumentList) masterQueryRsp.get("response");
assertEquals(501, masterQueryResult.getNumFound());
// NOTE: this test is wierd, we want to verify it DOESNT replicate...
// for now, add a sleep for this.., but the logic is wierd.
Thread.sleep(3000);
//get docs from slave and check if number is equal to master
slaveQueryRsp = query("*:*", slaveClient);
//get docs from slave and check if number is not equal to master; polling is disabled
slaveQueryRsp = rQuery(500, "*:*", slaveClient);
slaveQueryResult = (SolrDocumentList) slaveQueryRsp.get("response");
assertEquals(500, slaveQueryResult.getNumFound());
//get docs from slave and check if number is equal to master
slaveQueryRsp = query("*:*", masterClient);
slaveQueryResult = (SolrDocumentList) slaveQueryRsp.get("response");
assertEquals(501, slaveQueryResult.getNumFound());
}
@Test
public void testSnapPullWithMasterUrl() throws Exception {
//change solrconfig on slave
//this has no entry for pollinginterval
@ -326,7 +346,7 @@ public class TestReplicationHandler extends TestCase {
masterClient.commit();
NamedList masterQueryRsp = query("*:*", masterClient);
NamedList masterQueryRsp = rQuery(500, "*:*", masterClient);
SolrDocumentList masterQueryResult = (SolrDocumentList) masterQueryRsp.get("response");
assertEquals(500, masterQueryResult.getNumFound());
@ -340,9 +360,9 @@ public class TestReplicationHandler extends TestCase {
} catch (IOException e) {
//e.printStackTrace();
}
Thread.sleep(3000);
//get docs from slave and check if number is equal to master
NamedList slaveQueryRsp = query("*:*", slaveClient);
NamedList slaveQueryRsp = rQuery(500, "*:*", slaveClient);
SolrDocumentList slaveQueryResult = (SolrDocumentList) slaveQueryRsp.get("response");
assertEquals(500, slaveQueryResult.getNumFound());
//compare results
@ -350,6 +370,7 @@ public class TestReplicationHandler extends TestCase {
assertEquals(null, cmp);
}
@Test
public void testReplicateAfterStartup() throws Exception {
//stop slave
slaveJetty.stop();
@ -360,7 +381,7 @@ public class TestReplicationHandler extends TestCase {
masterClient.commit();
NamedList masterQueryRsp = query("*:*", masterClient);
NamedList masterQueryRsp = rQuery(500, "*:*", masterClient);
SolrDocumentList masterQueryResult = (SolrDocumentList) masterQueryRsp.get("response");
assertEquals(500, masterQueryResult.getNumFound());
@ -379,10 +400,8 @@ public class TestReplicationHandler extends TestCase {
slaveJetty = createJetty(slave);
slaveClient = createNewSolrServer(slaveJetty.getLocalPort());
//sleep for pollinterval time 3s, to let slave pull data.
Thread.sleep(3000);
//get docs from slave and check if number is equal to master
NamedList slaveQueryRsp = query("*:*", slaveClient);
NamedList slaveQueryRsp = rQuery(500, "*:*", slaveClient);
SolrDocumentList slaveQueryResult = (SolrDocumentList) slaveQueryRsp.get("response");
assertEquals(500, slaveQueryResult.getNumFound());
@ -392,6 +411,7 @@ public class TestReplicationHandler extends TestCase {
}
@Test
public void testReplicateAfterWrite2Slave() throws Exception {
//add 50 docs to master
int nDocs = 50;
@ -410,13 +430,13 @@ public class TestReplicationHandler extends TestCase {
masterClient.commit();
NamedList masterQueryRsp = query("*:*", masterClient);
NamedList masterQueryRsp = rQuery(50, "*:*", masterClient);
SolrDocumentList masterQueryResult = (SolrDocumentList) masterQueryRsp.get("response");
assertEquals(nDocs, masterQueryResult.getNumFound());
// Make sure that both the index version and index generation on the slave is
// higher than that of the master, just to make the test harder.
Thread.sleep(100);
index(slaveClient, "id", 551, "name", "name = " + 551);
slaveClient.commit(true, true);
index(slaveClient, "id", 552, "name", "name = " + 552);
@ -429,7 +449,7 @@ public class TestReplicationHandler extends TestCase {
slaveClient.commit(true, true);
//this doc is added to slave so it should show an item w/ that result
NamedList slaveQueryRsp = query("id:555", slaveClient);
NamedList slaveQueryRsp = rQuery(1, "id:555", slaveClient);
SolrDocumentList slaveQueryResult = (SolrDocumentList) slaveQueryRsp.get("response");
assertEquals(1, slaveQueryResult.getNumFound());
@ -442,14 +462,13 @@ public class TestReplicationHandler extends TestCase {
//e.printStackTrace();
}
//sleep for pollinterval time 3s, to let slave pull data.
Thread.sleep(3000);
//the slave should have done a full copy of the index so the doc with id:555 should not be there in the slave now
slaveQueryRsp = query("id:555", slaveClient);
slaveQueryRsp = rQuery(0, "id:555", slaveClient);
slaveQueryResult = (SolrDocumentList) slaveQueryRsp.get("response");
assertEquals(0, slaveQueryResult.getNumFound());
}
@Test
public void testBackup() throws Exception {
masterJetty.stop();
@ -543,22 +562,24 @@ public class TestReplicationHandler extends TestCase {
});
assertEquals(1, files.length);
File snapDir = files[0];
IndexSearcher searcher = new IndexSearcher(new SimpleFSDirectory(snapDir.getAbsoluteFile(), null), true);
Directory dir = new SimpleFSDirectory(snapDir.getAbsoluteFile());
IndexSearcher searcher = new IndexSearcher(dir, true);
TopDocs hits = searcher.search(new MatchAllDocsQuery(), 1);
assertEquals(500, hits.totalHits);
searcher.close();
dir.close();
}
/* character copy of file using UTF-8 */
void copyFile(File src, File dst) throws IOException {
private static void copyFile(File src, File dst) throws IOException {
copyFile(src, dst, null);
}
/**
* character copy of file using UTF-8. If port is non-null, will be substituted any time "TEST_PORT" is found.
*/
private void copyFile(File src, File dst, Integer port) throws IOException {
private static void copyFile(File src, File dst, Integer port) throws IOException {
BufferedReader in = new BufferedReader(new FileReader(src));
Writer out = new FileWriter(dst);
@ -572,12 +593,13 @@ public class TestReplicationHandler extends TestCase {
out.close();
}
private class SolrInstance extends AbstractSolrTestCase {
private static class SolrInstance {
String name;
Integer masterPort;
File homeDir;
File confDir;
File dataDir;
/**
* if masterPort is null, this instance is a master -- otherwise this instance is a slave, and assumes the master is
@ -592,7 +614,6 @@ public class TestReplicationHandler extends TestCase {
return homeDir.toString();
}
@Override
public String getSchemaFile() {
return CONF_DIR + "schema-replication1.xml";
}
@ -605,7 +626,6 @@ public class TestReplicationHandler extends TestCase {
return dataDir.toString();
}
@Override
public String getSolrConfigFile() {
String fname = "";
if (null == masterPort)
@ -616,7 +636,6 @@ public class TestReplicationHandler extends TestCase {
}
public void setUp() throws Exception {
super.setUp();
System.setProperty("solr.test.sys.prop1", "propone");
System.setProperty("solr.test.sys.prop2", "proptwo");
@ -645,7 +664,6 @@ public class TestReplicationHandler extends TestCase {
}
public void tearDown() throws Exception {
super.tearDown();
AbstractSolrTestCase.recurseDelete(homeDir);
}
}