SOLR-11673: Slave doesn't commit empty index if new index appears on master by deafult.

This commit is contained in:
Mikhail Khludnev 2018-04-01 16:47:23 +03:00
parent d98138ac03
commit 7a920cb0f6
4 changed files with 55 additions and 9 deletions

View File

@ -61,6 +61,10 @@ Upgrade Notes
This is a server side change only and clients using SolrJ won't need any changes. Clients can still use any logging This is a server side change only and clients using SolrJ won't need any changes. Clients can still use any logging
implementation which is compatible with SLF4J. implementation which is compatible with SLF4J.
* SOLR-11673: Slave doesn't commit empty index when completely new index is detected on master during replication.
To return the previous behavior pass false to skipCommitOnMasterVersionZero in slave section of replication
handler configuration, or pass it to the fetchindex command.
New Features New Features
---------------------- ----------------------
@ -95,6 +99,9 @@ Bug Fixes
document. Multithreaded test for AtomicUpdateRequestProcessor was also beefed up and fixed. document. Multithreaded test for AtomicUpdateRequestProcessor was also beefed up and fixed.
(Ishan Chattopadhyaya, Noble Paul, Amrit Sarkar, shalin) (Ishan Chattopadhyaya, Noble Paul, Amrit Sarkar, shalin)
* SOLR-11673: By default slave doesn't commit empty index when completely new index appears on master.
See Upgrade Notes to find a way to get back to the previous behavior. (Mikhail Khludnev)
Optimizations Optimizations
---------------------- ----------------------

View File

@ -16,6 +16,31 @@
*/ */
package org.apache.solr.handler; package org.apache.solr.handler;
import static org.apache.solr.common.params.CommonParams.JAVABIN;
import static org.apache.solr.common.params.CommonParams.NAME;
import static org.apache.solr.handler.ReplicationHandler.ALIAS;
import static org.apache.solr.handler.ReplicationHandler.CHECKSUM;
import static org.apache.solr.handler.ReplicationHandler.CMD_DETAILS;
import static org.apache.solr.handler.ReplicationHandler.CMD_GET_FILE;
import static org.apache.solr.handler.ReplicationHandler.CMD_GET_FILE_LIST;
import static org.apache.solr.handler.ReplicationHandler.CMD_INDEX_VERSION;
import static org.apache.solr.handler.ReplicationHandler.COMMAND;
import static org.apache.solr.handler.ReplicationHandler.COMPRESSION;
import static org.apache.solr.handler.ReplicationHandler.CONF_FILES;
import static org.apache.solr.handler.ReplicationHandler.CONF_FILE_SHORT;
import static org.apache.solr.handler.ReplicationHandler.EXTERNAL;
import static org.apache.solr.handler.ReplicationHandler.FETCH_FROM_LEADER;
import static org.apache.solr.handler.ReplicationHandler.FILE;
import static org.apache.solr.handler.ReplicationHandler.FILE_STREAM;
import static org.apache.solr.handler.ReplicationHandler.GENERATION;
import static org.apache.solr.handler.ReplicationHandler.INTERNAL;
import static org.apache.solr.handler.ReplicationHandler.MASTER_URL;
import static org.apache.solr.handler.ReplicationHandler.OFFSET;
import static org.apache.solr.handler.ReplicationHandler.SIZE;
import static org.apache.solr.handler.ReplicationHandler.SKIP_COMMIT_ON_MASTER_VERSION_ZERO;
import static org.apache.solr.handler.ReplicationHandler.TLOG_FILE;
import static org.apache.solr.handler.ReplicationHandler.TLOG_FILES;
import java.io.File; import java.io.File;
import java.io.FileNotFoundException; import java.io.FileNotFoundException;
import java.io.FileOutputStream; import java.io.FileOutputStream;
@ -86,7 +111,7 @@ import org.apache.solr.core.DirectoryFactory;
import org.apache.solr.core.DirectoryFactory.DirContext; import org.apache.solr.core.DirectoryFactory.DirContext;
import org.apache.solr.core.IndexDeletionPolicyWrapper; import org.apache.solr.core.IndexDeletionPolicyWrapper;
import org.apache.solr.core.SolrCore; import org.apache.solr.core.SolrCore;
import org.apache.solr.handler.ReplicationHandler.*; import org.apache.solr.handler.ReplicationHandler.FileInfo;
import org.apache.solr.request.LocalSolrQueryRequest; import org.apache.solr.request.LocalSolrQueryRequest;
import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.search.SolrIndexSearcher; import org.apache.solr.search.SolrIndexSearcher;
@ -99,13 +124,10 @@ import org.apache.solr.util.FileUtils;
import org.apache.solr.util.PropertiesOutputStream; import org.apache.solr.util.PropertiesOutputStream;
import org.apache.solr.util.RTimer; import org.apache.solr.util.RTimer;
import org.apache.solr.util.RefCounted; import org.apache.solr.util.RefCounted;
import org.apache.solr.util.TestInjection;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import static org.apache.solr.common.params.CommonParams.JAVABIN;
import static org.apache.solr.common.params.CommonParams.NAME;
import static org.apache.solr.handler.ReplicationHandler.*;
import com.google.common.base.Strings; import com.google.common.base.Strings;
/** /**
@ -167,7 +189,7 @@ public class IndexFetcher {
private boolean downloadTlogFiles = false; private boolean downloadTlogFiles = false;
private boolean skipCommitOnMasterVersionZero; private boolean skipCommitOnMasterVersionZero = true;
private static final String INTERRUPT_RESPONSE_MESSAGE = "Interrupted while waiting for modify lock"; private static final String INTERRUPT_RESPONSE_MESSAGE = "Interrupted while waiting for modify lock";
@ -453,6 +475,7 @@ public class IndexFetcher {
} finally { } finally {
iw.decref(); iw.decref();
} }
assert TestInjection.injectDelayBeforeSlaveCommitRefresh();
if (skipCommitOnMasterVersionZero) { if (skipCommitOnMasterVersionZero) {
openNewSearcherAndUpdateCommitPoint(); openNewSearcherAndUpdateCommitPoint();
} else { } else {

View File

@ -144,6 +144,8 @@ public class TestInjection {
private static AtomicInteger countPrepRecoveryOpPauseForever = new AtomicInteger(0); private static AtomicInteger countPrepRecoveryOpPauseForever = new AtomicInteger(0);
public static Integer delayBeforeSlaveCommitRefresh=null;
public static void reset() { public static void reset() {
nonGracefullClose = null; nonGracefullClose = null;
failReplicaRequests = null; failReplicaRequests = null;
@ -158,6 +160,7 @@ public class TestInjection {
waitForReplicasInSync = "true:60"; waitForReplicasInSync = "true:60";
failIndexFingerprintRequests = null; failIndexFingerprintRequests = null;
wrongIndexFingerprint = null; wrongIndexFingerprint = null;
delayBeforeSlaveCommitRefresh = null;
for (Timer timer : timers) { for (Timer timer : timers) {
timer.cancel(); timer.cancel();
@ -455,4 +458,16 @@ public class TestInjection {
return new Pair<>(Boolean.parseBoolean(val), Integer.parseInt(percent)); return new Pair<>(Boolean.parseBoolean(val), Integer.parseInt(percent));
} }
public static boolean injectDelayBeforeSlaveCommitRefresh() {
if (delayBeforeSlaveCommitRefresh!=null) {
try {
log.info("Pausing IndexFetcher for {}ms", delayBeforeSlaveCommitRefresh);
Thread.sleep(delayBeforeSlaveCommitRefresh);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
}
return true;
}
} }

View File

@ -70,6 +70,7 @@ import org.apache.solr.core.SolrCore;
import org.apache.solr.core.StandardDirectoryFactory; import org.apache.solr.core.StandardDirectoryFactory;
import org.apache.solr.core.snapshots.SolrSnapshotMetaDataManager; import org.apache.solr.core.snapshots.SolrSnapshotMetaDataManager;
import org.apache.solr.util.FileUtils; import org.apache.solr.util.FileUtils;
import org.apache.solr.util.TestInjection;
import org.junit.After; import org.junit.After;
import org.junit.Before; import org.junit.Before;
import org.junit.BeforeClass; import org.junit.BeforeClass;
@ -494,8 +495,10 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {
} }
@Test @Test
@BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-11673")
public void doTestIndexAndConfigReplication() throws Exception { public void doTestIndexAndConfigReplication() throws Exception {
TestInjection.delayBeforeSlaveCommitRefresh = random().nextInt(10);
clearIndexWithReplication(); clearIndexWithReplication();
nDocs--; nDocs--;
@ -550,7 +553,6 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {
slaveJetty = createJetty(slave); slaveJetty = createJetty(slave);
slaveClient.close(); slaveClient.close();
slaveClient = createNewSolrClient(slaveJetty.getLocalPort()); slaveClient = createNewSolrClient(slaveJetty.getLocalPort());
//add a doc with new field and commit on master to trigger index fetch from slave. //add a doc with new field and commit on master to trigger index fetch from slave.
index(masterClient, "id", "2000", "name", "name = " + 2000, "newname", "newname = " + 2000); index(masterClient, "id", "2000", "name", "name = " + 2000, "newname", "newname = " + 2000);
masterClient.commit(); masterClient.commit();
@ -567,7 +569,6 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {
checkForSingleIndex(masterJetty); checkForSingleIndex(masterJetty);
checkForSingleIndex(slaveJetty, true); checkForSingleIndex(slaveJetty, true);
} }
@Test @Test