Fix all solr tests to pass w/ -Dtests.dups=2 -Dtests.jvms=1

This commit is contained in:
Chris Hostetter 2019-05-29 10:27:15 -07:00
parent 6ca372fcbb
commit c2b92f3015
18 changed files with 105 additions and 90 deletions

View File

@ -69,11 +69,17 @@ public class LegacyNoFacetTest extends LegacyAbstractAnalyticsTest {
defaults.put("string_sd", "str0");
intTestStart = new ArrayList<>();
intMissing = 0;
longTestStart = new ArrayList<>();
longMissing = 0;
floatTestStart = new ArrayList<>();
floatMissing = 0;
doubleTestStart = new ArrayList<>();
doubleMissing = 0;
dateTestStart = new ArrayList<>();
dateMissing = 0;
stringTestStart = new ArrayList<>();
stringMissing = 0;
for (int j = 0; j < NUM_LOOPS; ++j) {
int i = j%INT;

View File

@ -22,6 +22,7 @@ import java.util.List;
import org.apache.solr.client.solrj.request.UpdateRequest;
import org.apache.solr.common.util.NamedList;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@ -34,10 +35,18 @@ public class LegacyQueryFacetCloudTest extends LegacyAbstractAnalyticsFacetCloud
private static final int STRING = 7;
private static final int NUM_LOOPS = 100;
private static ArrayList<ArrayList<Integer>> int1TestStart = new ArrayList<>();
private static ArrayList<ArrayList<Integer>> int2TestStart = new ArrayList<>();
private static ArrayList<ArrayList<Long>> longTestStart = new ArrayList<>();
private static ArrayList<ArrayList<Float>> floatTestStart = new ArrayList<>();
private static final ArrayList<ArrayList<Integer>> int1TestStart = new ArrayList<>();
private static final ArrayList<ArrayList<Integer>> int2TestStart = new ArrayList<>();
private static final ArrayList<ArrayList<Long>> longTestStart = new ArrayList<>();
private static final ArrayList<ArrayList<Float>> floatTestStart = new ArrayList<>();
@After
public void afterTest() throws Exception {
int1TestStart.clear();
int2TestStart.clear();
longTestStart.clear();
floatTestStart.clear();
}
@Before
public void beforeTest() throws Exception {

View File

@ -48,6 +48,15 @@ public class BufferStore implements Store {
private final AtomicLong shardBuffercacheAllocate;
private final AtomicLong shardBuffercacheLost;
/**
* For testing only
*
* @lucene.internal
*/
static void clearBufferStores() {
bufferStores.clear();
}
public synchronized static void initNewBuffer(int bufferSize, long totalAmount) {
initNewBuffer(bufferSize, totalAmount, null);
}

View File

@ -56,7 +56,7 @@ public class TestCloudPhrasesIdentificationComponent extends SolrCloudTestCase {
/** A basic client for operations at the cloud level, default collection will be set */
private static CloudSolrClient CLOUD_CLIENT;
/** One client per node */
private static ArrayList<HttpSolrClient> CLIENTS = new ArrayList<>(5);
private static final ArrayList<HttpSolrClient> CLIENTS = new ArrayList<>(5);
@BeforeClass
private static void createMiniSolrCloudCluster() throws Exception {
@ -114,7 +114,7 @@ public class TestCloudPhrasesIdentificationComponent extends SolrCloudTestCase {
for (HttpSolrClient client : CLIENTS) {
client.close();
}
CLIENTS = null;
CLIENTS.clear();
}
public void testBasicPhrases() throws Exception {

View File

@ -58,7 +58,7 @@ public class TestCloudPseudoReturnFields extends SolrCloudTestCase {
/** A basic client for operations at the cloud level, default collection will be set */
private static CloudSolrClient CLOUD_CLIENT;
/** One client per node */
private static ArrayList<HttpSolrClient> CLIENTS = new ArrayList<>(5);
private static final ArrayList<HttpSolrClient> CLIENTS = new ArrayList<>(5);
@BeforeClass
private static void createMiniSolrCloudCluster() throws Exception {
@ -114,7 +114,7 @@ public class TestCloudPseudoReturnFields extends SolrCloudTestCase {
for (HttpSolrClient client : CLIENTS) {
client.close();
}
CLIENTS = null;
CLIENTS.clear();
}
public void testMultiValued() throws Exception {

View File

@ -66,7 +66,7 @@ public class TestRandomFlRTGCloud extends SolrCloudTestCase {
/** A basic client for operations at the cloud level, default collection will be set */
private static CloudSolrClient CLOUD_CLIENT;
/** One client per node */
private static List<HttpSolrClient> CLIENTS = Collections.synchronizedList(new ArrayList<>(5));
private static final List<HttpSolrClient> CLIENTS = Collections.synchronizedList(new ArrayList<>(5));
/** Always included in fl so we can vet what doc we're looking at */
private static final FlValidator ID_VALIDATOR = new SimpleFieldValueValidator("id");
@ -159,7 +159,7 @@ public class TestRandomFlRTGCloud extends SolrCloudTestCase {
for (HttpSolrClient client : CLIENTS) {
client.close();
}
CLIENTS = null;
CLIENTS.clear();
}
/**

View File

@ -110,7 +110,7 @@ public class HttpTriggerListenerTest extends SolrCloudTestCase {
response = solrClient.request(req);
assertEquals(response.get("result").toString(), "success");
assertEquals(requests.toString(), 0, requests.size());
assertEquals(mockService.requests.toString(), 0, mockService.requests.size());
cluster.startJettySolrRunner();
cluster.waitForAllNodes(30);
@ -119,25 +119,25 @@ public class HttpTriggerListenerTest extends SolrCloudTestCase {
Thread.sleep(5000);
assertEquals(requests.toString(), 4, requests.size());
requests.forEach(s -> assertTrue(s.contains("Content-Type: application/json")));
requests.forEach(s -> assertTrue(s.contains("X-Foo: foo")));
requests.forEach(s -> assertTrue(s.contains("source=node_added_trigger")));
requests.forEach(s -> assertTrue(s.contains("type=NODEADDED")));
assertEquals(mockService.requests.toString(), 4, mockService.requests.size());
mockService.requests.forEach(s -> assertTrue(s.contains("Content-Type: application/json")));
mockService.requests.forEach(s -> assertTrue(s.contains("X-Foo: foo")));
mockService.requests.forEach(s -> assertTrue(s.contains("source=node_added_trigger")));
mockService.requests.forEach(s -> assertTrue(s.contains("type=NODEADDED")));
String request = requests.get(0);
String request = mockService.requests.get(0);
assertTrue(request, request.startsWith("/foo/test/STARTED"));
assertTrue(request, request.contains("actionName=,")); // empty actionName
request = requests.get(1);
request = mockService.requests.get(1);
assertTrue(request, request.startsWith("/foo/test/BEFORE_ACTION"));
assertTrue(request, request.contains("actionName=test,")); // actionName
request = requests.get(2);
request = mockService.requests.get(2);
assertTrue(request, request.startsWith("/foo/test/AFTER_ACTION"));
assertTrue(request, request.contains("actionName=test,")); // actionName
request = requests.get(3);
request = mockService.requests.get(3);
assertTrue(request, request.startsWith("/foo/test/SUCCEEDED"));
assertTrue(request, request.contains("actionName=,")); // empty actionName
}
@ -150,11 +150,9 @@ public class HttpTriggerListenerTest extends SolrCloudTestCase {
}
}
static List<String> requests = new ArrayList<>();
private static class MockService extends Thread {
Server server;
public final List<String> requests = new ArrayList<>();
private Server server;
public void start() {
server = new Server(new InetSocketAddress("localhost", 0));

View File

@ -22,8 +22,6 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
@ -61,11 +59,9 @@ public class MetricTriggerIntegrationTest extends SolrCloudTestCase {
private static final TimeSource timeSource = TimeSource.NANO_TIME;
static Map<String, List<CapturedEvent>> listenerEvents = new HashMap<>();
static CountDownLatch listenerCreated = new CountDownLatch(1);
static final Map<String, List<CapturedEvent>> listenerEvents = new HashMap<>();
private static CountDownLatch triggerFiredLatch;
private static int waitForSeconds = 1;
private static Set<TriggerEvent> events = ConcurrentHashMap.newKeySet();
@BeforeClass
public static void setupCluster() throws Exception {
@ -77,6 +73,7 @@ public class MetricTriggerIntegrationTest extends SolrCloudTestCase {
CloudTestUtils.waitForTriggerToBeScheduled(cluster.getOpenOverseer().getSolrCloudManager(), ".scheduled_maintenance");
CloudTestUtils.suspendTrigger(cluster.getOpenOverseer().getSolrCloudManager(), ".scheduled_maintenance");
listenerEvents.clear();
triggerFiredLatch = new CountDownLatch(1);
}
@ -211,7 +208,6 @@ public class MetricTriggerIntegrationTest extends SolrCloudTestCase {
@Override
public void process(TriggerEvent event, ActionContext context) throws Exception {
try {
events.add(event);
long currentTimeNanos = context.getCloudManager().getTimeSource().getTimeNs();
long eventTimeNanos = event.getEventTime();
long waitForNanos = TimeUnit.NANOSECONDS.convert(waitForSeconds, TimeUnit.SECONDS) - WAIT_FOR_DELTA_NANOS;
@ -231,7 +227,6 @@ public class MetricTriggerIntegrationTest extends SolrCloudTestCase {
@Override
public void configure(SolrResourceLoader loader, SolrCloudManager cloudManager, AutoScalingConfig.TriggerListenerConfig config) throws TriggerValidationException {
super.configure(loader, cloudManager, config);
listenerCreated.countDown();
timeSource = cloudManager.getTimeSource();
}

View File

@ -52,8 +52,8 @@ public class ScheduledTriggerIntegrationTest extends SolrCloudTestCase {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private static CountDownLatch triggerFiredLatch;
private static Set<TriggerEvent> events = ConcurrentHashMap.newKeySet();
private static AtomicReference<Map<String, Object>> actionContextPropertiesRef = new AtomicReference<>();
private static final Set<TriggerEvent> events = ConcurrentHashMap.newKeySet();
private static final AtomicReference<Map<String, Object>> actionContextPropertiesRef = new AtomicReference<>();
@Before
public void setupCluster() throws Exception {
@ -71,6 +71,8 @@ public class ScheduledTriggerIntegrationTest extends SolrCloudTestCase {
@After
public void afterTest() throws Exception {
shutdownCluster();
events.clear();
actionContextPropertiesRef.set(null);
}
@Test

View File

@ -22,8 +22,6 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
@ -52,18 +50,22 @@ import static org.apache.solr.cloud.autoscaling.TriggerIntegrationTest.WAIT_FOR_
@LogLevel("org.apache.solr.cloud.autoscaling=DEBUG;org.apache.solr.client.solrj.cloud.autoscaling=DEBUG")
public class TriggerCooldownIntegrationTest extends SolrCloudTestCase {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
static Map<String, List<CapturedEvent>> listenerEvents = new HashMap<>();
static CountDownLatch listenerCreated = new CountDownLatch(1);
static boolean failDummyAction = false;
private static CountDownLatch actionConstructorCalled = new CountDownLatch(1);
private static CountDownLatch actionInitCalled = new CountDownLatch(1);
private static final int waitForSeconds = 1;
private static final Map<String, List<CapturedEvent>> listenerEvents = new HashMap<>();
private static CountDownLatch triggerFiredLatch = new CountDownLatch(1);
private static int waitForSeconds = 1;
private static AtomicBoolean triggerFired = new AtomicBoolean();
private static Set<TriggerEvent> events = ConcurrentHashMap.newKeySet();
private static final AtomicBoolean triggerFired = new AtomicBoolean();
private static final void resetTriggerAndListenerState() {
// reset the trigger and captured events
listenerEvents.clear();
triggerFiredLatch = new CountDownLatch(1);
triggerFired.compareAndSet(true, false);
}
@BeforeClass
public static void setupCluster() throws Exception {
resetTriggerAndListenerState();
configureCluster(2)
.addConfig("conf", configset("cloud-minimal"))
.configure();
@ -76,8 +78,6 @@ public class TriggerCooldownIntegrationTest extends SolrCloudTestCase {
@Test
public void testCooldown() throws Exception {
CloudSolrClient solrClient = cluster.getSolrClient();
failDummyAction = false;
waitForSeconds = 1;
String setTriggerCommand = "{" +
"'set-trigger' : {" +
"'name' : 'node_added_cooldown_trigger'," +
@ -105,9 +105,6 @@ public class TriggerCooldownIntegrationTest extends SolrCloudTestCase {
response = solrClient.request(req);
assertEquals(response.get("result").toString(), "success");
listenerCreated = new CountDownLatch(1);
listenerEvents.clear();
JettySolrRunner newNode = cluster.startJettySolrRunner();
cluster.waitForAllNodes(30);
boolean await = triggerFiredLatch.await(20, TimeUnit.SECONDS);
@ -121,10 +118,7 @@ public class TriggerCooldownIntegrationTest extends SolrCloudTestCase {
assertTrue(capturedEvents.toString(), capturedEvents.size() > 0);
long prevTimestamp = capturedEvents.get(capturedEvents.size() - 1).timestamp;
// reset the trigger and captured events
listenerEvents.clear();
triggerFiredLatch = new CountDownLatch(1);
triggerFired.compareAndSet(true, false);
resetTriggerAndListenerState();
JettySolrRunner newNode2 = cluster.startJettySolrRunner();
await = triggerFiredLatch.await(20, TimeUnit.SECONDS);
@ -153,10 +147,7 @@ public class TriggerCooldownIntegrationTest extends SolrCloudTestCase {
req = AutoScalingRequest.create(SolrRequest.METHOD.GET, null);
response = solrClient.request(req);
// reset the trigger and captured events
listenerEvents.clear();
triggerFiredLatch = new CountDownLatch(1);
triggerFired.compareAndSet(true, false);
resetTriggerAndListenerState();
JettySolrRunner newNode3 = cluster.startJettySolrRunner();
await = triggerFiredLatch.await(20, TimeUnit.SECONDS);
@ -187,14 +178,13 @@ public class TriggerCooldownIntegrationTest extends SolrCloudTestCase {
public static class TestTriggerAction extends TriggerActionBase {
public TestTriggerAction() {
actionConstructorCalled.countDown();
// No-Op
}
@Override
public void process(TriggerEvent event, ActionContext actionContext) {
try {
if (triggerFired.compareAndSet(false, true)) {
events.add(event);
long currentTimeNanos = actionContext.getCloudManager().getTimeSource().getTimeNs();
long eventTimeNanos = event.getEventTime();
long waitForNanos = TimeUnit.NANOSECONDS.convert(waitForSeconds, TimeUnit.SECONDS) - WAIT_FOR_DELTA_NANOS;
@ -214,7 +204,6 @@ public class TriggerCooldownIntegrationTest extends SolrCloudTestCase {
@Override
public void init() throws Exception {
log.info("TestTriggerAction init");
actionInitCalled.countDown();
super.init();
}
}
@ -224,7 +213,6 @@ public class TriggerCooldownIntegrationTest extends SolrCloudTestCase {
@Override
public void configure(SolrResourceLoader loader, SolrCloudManager cloudManager, AutoScalingConfig.TriggerListenerConfig config) throws TriggerValidationException {
super.configure(loader, cloudManager, config);
listenerCreated.countDown();
timeSource = cloudManager.getTimeSource();
}

View File

@ -25,6 +25,7 @@ import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.response.SolrQueryResponse;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
@ -38,16 +39,24 @@ public class MetricsHandlerTest extends SolrTestCaseJ4 {
initCore("solrconfig-minimal.xml", "schema.xml");
h.getCoreContainer().waitForLoadingCoresToFinish(30000);
// manually register some metrics in solr.jvm and solr.jetty - TestHarness doesn't init them
Counter c = h.getCoreContainer().getMetricManager().counter(null, "solr.jvm", "foo");
// manually register & seed some metrics in solr.jvm and solr.jetty for testing via handler
// (use "solrtest_" prefix just in case the jvm or jetty ads a "foo" metric at some point)
Counter c = h.getCoreContainer().getMetricManager().counter(null, "solr.jvm", "solrtest_foo");
c.inc();
c = h.getCoreContainer().getMetricManager().counter(null, "solr.jetty", "foo");
c = h.getCoreContainer().getMetricManager().counter(null, "solr.jetty", "solrtest_foo");
c.inc(2);
// test escapes
c = h.getCoreContainer().getMetricManager().counter(null, "solr.jetty", "foo:bar");
c = h.getCoreContainer().getMetricManager().counter(null, "solr.jetty", "solrtest_foo:bar");
c.inc(3);
}
@AfterClass
public static void cleanupMetrics() throws Exception {
h.getCoreContainer().getMetricManager().registry("solr.jvm" ).remove("solrtest_foo");
h.getCoreContainer().getMetricManager().registry("solr.jetty").remove("solrtest_foo");
h.getCoreContainer().getMetricManager().registry("solr.jetty").remove("solrtest_foo:bar");
}
@Test
public void test() throws Exception {
MetricsHandler handler = new MetricsHandler(h.getCoreContainer());
@ -257,7 +266,7 @@ public class MetricsHandlerTest extends SolrTestCaseJ4 {
assertNotNull(val);
assertTrue(val instanceof Number);
String key3 = "solr.jetty:foo\\:bar";
String key3 = "solr.jetty:solrtest_foo\\:bar";
resp = new SolrQueryResponse();
handler.handleRequestBody(req(CommonParams.QT, "/admin/metrics", CommonParams.WT, "json",
MetricsHandler.KEY_PARAM, key3), resp);

View File

@ -51,6 +51,8 @@ public class TestOverriddenPrefixQueryForCustomFieldType extends SolrTestCaseJ4
super.setUp();
clearIndex();
assertU(commit());
otherCounts=0;
counts = new int[2];
}
public void createIndex(int nDocs) {

View File

@ -102,6 +102,8 @@ public class RangeFacetCloudTest extends SolrCloudTestCase {
final int numDocs = atLeast(1000);
final int maxTermId = atLeast(TERM_VALUES_RANDOMIZER);
// clear the RANGE_MODEL
Arrays.fill(RANGE_MODEL, 0);
// seed the TERM_MODEL Maps so we don't have null check later
for (int i = 0; i < NUM_RANGE_VALUES; i++) {
TERM_MODEL[i] = new LinkedHashMap<>();

View File

@ -83,7 +83,7 @@ public class TestCloudJSONFacetJoinDomain extends SolrCloudTestCase {
/** A basic client for operations at the cloud level, default collection will be set */
private static CloudSolrClient CLOUD_CLIENT;
/** One client per node */
private static ArrayList<HttpSolrClient> CLIENTS = new ArrayList<>(5);
private static final ArrayList<HttpSolrClient> CLIENTS = new ArrayList<>(5);
@BeforeClass
private static void createMiniSolrCloudCluster() throws Exception {
@ -188,7 +188,7 @@ public class TestCloudJSONFacetJoinDomain extends SolrCloudTestCase {
for (HttpSolrClient client : CLIENTS) {
client.close();
}
CLIENTS = null;
CLIENTS.clear();
}
/** Sanity check that malformed requests produce errors */

View File

@ -94,7 +94,7 @@ public class TestCloudJSONFacetSKG extends SolrCloudTestCase {
/** A basic client for operations at the cloud level, default collection will be set */
private static CloudSolrClient CLOUD_CLIENT;
/** One client per node */
private static ArrayList<HttpSolrClient> CLIENTS = new ArrayList<>(5);
private static final ArrayList<HttpSolrClient> CLIENTS = new ArrayList<>(5);
@BeforeClass
private static void createMiniSolrCloudCluster() throws Exception {
@ -210,7 +210,7 @@ public class TestCloudJSONFacetSKG extends SolrCloudTestCase {
for (HttpSolrClient client : CLIENTS) {
client.close();
}
CLIENTS = null;
CLIENTS.clear();
}
/**

View File

@ -23,6 +23,7 @@ import org.apache.lucene.util.TestUtil;
import org.apache.solr.SolrTestCase;
import org.apache.solr.metrics.MetricsMap;
import org.apache.solr.metrics.SolrMetricManager;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@ -46,6 +47,11 @@ public class BufferStoreTest extends SolrTestCase {
store = BufferStore.instance(blockSize);
}
@After
public void clearBufferStores() {
BufferStore.clearBufferStores();
}
@Test
public void testBufferTakePut() {
byte[] b1 = store.takeBuffer(blockSize);

View File

@ -90,8 +90,8 @@ public class AddBlockUpdateTest extends SolrTestCaseJ4 {
private static final String parent = "parent_s";
private static final String type = "type_s";
private final static AtomicInteger counter = new AtomicInteger();
private static ExecutorService exe;
private static AtomicInteger counter = new AtomicInteger();
private static boolean cachedMode;
private static XMLInputFactory inputFactory;
@ -117,7 +117,7 @@ public class AddBlockUpdateTest extends SolrTestCaseJ4 {
rarely() ? ExecutorUtil.newMDCAwareFixedThreadPool(atLeast(2), new DefaultSolrThreadFactory("AddBlockUpdateTest")) : ExecutorUtil
.newMDCAwareCachedThreadPool(new DefaultSolrThreadFactory("AddBlockUpdateTest"));
counter.set(0);
initCore("solrconfig.xml", "schema15.xml");
}
@ -157,7 +157,6 @@ public class AddBlockUpdateTest extends SolrTestCaseJ4 {
exe = null;
inputFactory = null;
counter = null;
}
@Test

View File

@ -32,9 +32,7 @@ import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.cloud.SolrCloudTestCase;
import org.apache.solr.common.util.ExecutorUtil;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -44,34 +42,26 @@ public class TestCollectionStateWatchers extends SolrCloudTestCase {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private static final int CLUSTER_SIZE = 4;
private static final ExecutorService executor = ExecutorUtil.newMDCAwareCachedThreadPool("backgroundWatchers");
private static final int MAX_WAIT_TIMEOUT = 30;
@BeforeClass
public static void startCluster() throws Exception {
}
@AfterClass
public static void shutdownBackgroundExecutors() {
executor.shutdown();
}
private ExecutorService executor = null;
@Before
public void prepareCluster() throws Exception {
configureCluster(CLUSTER_SIZE)
.addConfig("config", getFile("solrj/solr/collection1/conf").toPath())
.configure();
executor = ExecutorUtil.newMDCAwareCachedThreadPool("backgroundWatchers");
}
@After
public void tearDownCluster() throws Exception {
executor.shutdown();
shutdownCluster();
executor = null;
}
private static Future<Boolean> waitInBackground(String collection, long timeout, TimeUnit unit,
private Future<Boolean> waitInBackground(String collection, long timeout, TimeUnit unit,
CollectionStatePredicate predicate) {
return executor.submit(() -> {
try {
@ -83,7 +73,7 @@ public class TestCollectionStateWatchers extends SolrCloudTestCase {
});
}
private static void waitFor(String message, long timeout, TimeUnit unit, Callable<Boolean> predicate)
private void waitFor(String message, long timeout, TimeUnit unit, Callable<Boolean> predicate)
throws InterruptedException, ExecutionException {
Future<Boolean> future = executor.submit(() -> {
try {