Fix a racing condition in MockTransportService#addUnresponsiveRule where a request can be delayed even if the rule was removed.

Relates to #21129

Also properly reset DiscoveryWithServiceDisruptionsIT#disableBeforeIndexDeletion
This commit is contained in:
Boaz Leskes 2016-11-01 13:55:55 +01:00
parent 6e7e89159b
commit 523f7ea71e
2 changed files with 25 additions and 9 deletions

View File

@ -50,13 +50,13 @@ import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.discovery.zen.ElectMasterService; import org.elasticsearch.discovery.zen.ElectMasterService;
import org.elasticsearch.discovery.zen.ZenDiscovery;
import org.elasticsearch.discovery.zen.FaultDetection; import org.elasticsearch.discovery.zen.FaultDetection;
import org.elasticsearch.discovery.zen.MembershipAction; import org.elasticsearch.discovery.zen.MembershipAction;
import org.elasticsearch.discovery.zen.PublishClusterStateAction;
import org.elasticsearch.discovery.zen.UnicastZenPing;
import org.elasticsearch.discovery.zen.ZenDiscovery;
import org.elasticsearch.discovery.zen.ZenPing; import org.elasticsearch.discovery.zen.ZenPing;
import org.elasticsearch.discovery.zen.ZenPingService; import org.elasticsearch.discovery.zen.ZenPingService;
import org.elasticsearch.discovery.zen.UnicastZenPing;
import org.elasticsearch.discovery.zen.PublishClusterStateAction;
import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.env.NodeEnvironment;
import org.elasticsearch.indices.store.IndicesStoreIntegrationIT; import org.elasticsearch.indices.store.IndicesStoreIntegrationIT;
import org.elasticsearch.monitor.jvm.HotThreads; import org.elasticsearch.monitor.jvm.HotThreads;
@ -155,6 +155,12 @@ public class DiscoveryWithServiceDisruptionsIT extends ESIntegTestCase {
private boolean disableBeforeIndexDeletion; private boolean disableBeforeIndexDeletion;
@Before
public void setUp() throws Exception {
super.setUp();
disableBeforeIndexDeletion = false;
}
@Override @Override
public void setDisruptionScheme(ServiceDisruptionScheme scheme) { public void setDisruptionScheme(ServiceDisruptionScheme scheme) {
if (scheme instanceof NetworkDisruption && if (scheme instanceof NetworkDisruption &&

View File

@ -60,9 +60,9 @@ import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Queue; import java.util.Queue;
import java.util.Set; import java.util.Set;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.LinkedBlockingDeque;
import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicBoolean;
/** /**
@ -299,7 +299,8 @@ public final class MockTransportService extends TransportService {
final long startTime = System.currentTimeMillis(); final long startTime = System.currentTimeMillis();
addDelegate(transportAddress, new ClearableTransport(original) { addDelegate(transportAddress, new ClearableTransport(original) {
private final Queue<Runnable> requestsToSendWhenCleared = new ConcurrentLinkedQueue<>(); private final Queue<Runnable> requestsToSendWhenCleared = new LinkedBlockingDeque<Runnable>();
private boolean cleared = false;
TimeValue getDelay() { TimeValue getDelay() {
return new TimeValue(duration.millis() - (System.currentTimeMillis() - startTime)); return new TimeValue(duration.millis() - (System.currentTimeMillis() - startTime));
@ -386,15 +387,24 @@ public final class MockTransportService extends TransportService {
}; };
// store the request to send it once the rule is cleared. // store the request to send it once the rule is cleared.
requestsToSendWhenCleared.add(runnable); synchronized (this) {
if (cleared) {
threadPool.schedule(delay, ThreadPool.Names.GENERIC, runnable); runnable.run();
} else {
requestsToSendWhenCleared.add(runnable);
threadPool.schedule(delay, ThreadPool.Names.GENERIC, runnable);
}
}
} }
@Override @Override
public void clearRule() { public void clearRule() {
requestsToSendWhenCleared.forEach(Runnable::run); synchronized (this) {
assert cleared == false;
cleared = true;
requestsToSendWhenCleared.forEach(Runnable::run);
}
} }
}); });
} }