diff --git a/server/src/test/java/org/elasticsearch/bwcompat/RecoveryWithUnsupportedIndicesIT.java b/server/src/test/java/org/elasticsearch/bwcompat/RecoveryWithUnsupportedIndicesIT.java index 53efeb393e4..8ef27c8e939 100644 --- a/server/src/test/java/org/elasticsearch/bwcompat/RecoveryWithUnsupportedIndicesIT.java +++ b/server/src/test/java/org/elasticsearch/bwcompat/RecoveryWithUnsupportedIndicesIT.java @@ -85,7 +85,7 @@ public class RecoveryWithUnsupportedIndicesIT extends ESIntegTestCase { String indexName = "unsupported-2.4.5"; logger.info("Checking static index {}", indexName); - Settings nodeSettings = prepareBackwardsDataDir(getBwcIndicesPath().resolve(indexName + ".zip")); + Settings nodeSettings = prepareBackwardsDataDir(getDataPath("/indices/bwc").resolve(indexName + ".zip")); assertThat(expectThrows(Exception.class, () -> internalCluster().startNode(nodeSettings)) .getCause().getCause().getMessage(), containsString("Format version is not supported")); } diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/JoinTaskExecutorTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/JoinTaskExecutorTests.java index 35fa5786bbd..31e9a4e586e 100644 --- a/server/src/test/java/org/elasticsearch/cluster/coordination/JoinTaskExecutorTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/coordination/JoinTaskExecutorTests.java @@ -19,7 +19,6 @@ package org.elasticsearch.cluster.coordination; import org.elasticsearch.Version; -import org.elasticsearch.cluster.coordination.JoinTaskExecutor; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNode; diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java b/server/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java index 6948fd277c6..98664a56655 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java +++ b/server/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java @@ -90,6 +90,7 @@ import org.junit.Assert; import java.io.IOException; import java.io.UncheckedIOException; +import java.nio.file.DirectoryStream; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; @@ -101,6 +102,7 @@ import java.util.Locale; import java.util.concurrent.BrokenBarrierException; import java.util.concurrent.CountDownLatch; import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; @@ -956,4 +958,43 @@ public class IndexShardIT extends ESSingleNodeTestCase { assertThat(indexShard.getEngine(), instanceOf(NoOpEngine.class)); } } + + /** + * Asserts that there are no files in the specified path + */ + private void assertPathHasBeenCleared(Path path) { + logger.info("--> checking that [{}] has been cleared", path); + int count = 0; + StringBuilder sb = new StringBuilder(); + sb.append("["); + if (Files.exists(path)) { + try (DirectoryStream stream = Files.newDirectoryStream(path)) { + for (Path file : stream) { + // Skip files added by Lucene's ExtraFS + if (file.getFileName().toString().startsWith("extra")) { + continue; + } + logger.info("--> found file: [{}]", file.toAbsolutePath().toString()); + if (Files.isDirectory(file)) { + assertPathHasBeenCleared(file); + } else if (Files.isRegularFile(file)) { + count++; + sb.append(file.toAbsolutePath().toString()); + sb.append("\n"); + } + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + sb.append("]"); + assertThat(count + " files exist that should have been cleaned:\n" + sb.toString(), count, equalTo(0)); + } + + private static void assertAllIndicesRemovedAndDeletionCompleted(Iterable indicesServices) throws Exception { + for (IndicesService indicesService : indicesServices) { + assertBusy(() -> assertFalse(indicesService.iterator().hasNext()), 1, TimeUnit.MINUTES); + assertBusy(() -> assertFalse(indicesService.hasUncompletedPendingDeletes()), 1, TimeUnit.MINUTES); + } + } } diff --git a/server/src/test/java/org/elasticsearch/indices/mapping/UpdateMappingIntegrationIT.java b/server/src/test/java/org/elasticsearch/indices/mapping/UpdateMappingIntegrationIT.java index fab952b6581..19f7324db43 100644 --- a/server/src/test/java/org/elasticsearch/indices/mapping/UpdateMappingIntegrationIT.java +++ b/server/src/test/java/org/elasticsearch/indices/mapping/UpdateMappingIntegrationIT.java @@ -33,7 +33,11 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.index.IndexService; +import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.indices.IndicesService; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.InternalSettingsPlugin; @@ -45,6 +49,7 @@ import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.CyclicBarrier; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; @@ -57,6 +62,7 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcke import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertBlocked; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.notNullValue; public class UpdateMappingIntegrationIT extends ESIntegTestCase { @@ -285,4 +291,47 @@ public class UpdateMappingIntegrationIT extends ESIntegTestCase { } } + /** + * Waits until mappings for the provided fields exist on all nodes. Note, this waits for the current + * started shards and checks for concrete mappings. + */ + private void assertConcreteMappingsOnAll(final String index, final String type, final String... fieldNames) { + Set nodes = internalCluster().nodesInclude(index); + assertThat(nodes, Matchers.not(Matchers.emptyIterable())); + for (String node : nodes) { + IndicesService indicesService = internalCluster().getInstance(IndicesService.class, node); + IndexService indexService = indicesService.indexService(resolveIndex(index)); + assertThat("index service doesn't exists on " + node, indexService, notNullValue()); + MapperService mapperService = indexService.mapperService(); + for (String fieldName : fieldNames) { + MappedFieldType fieldType = mapperService.fullName(fieldName); + assertNotNull("field " + fieldName + " doesn't exists on " + node, fieldType); + } + } + assertMappingOnMaster(index, type, fieldNames); + } + + /** + * Waits for the given mapping type to exists on the master node. + */ + private void assertMappingOnMaster(final String index, final String type, final String... fieldNames) { + GetMappingsResponse response = client().admin().indices().prepareGetMappings(index).setTypes(type).get(); + ImmutableOpenMap mappings = response.getMappings().get(index); + assertThat(mappings, notNullValue()); + MappingMetaData mappingMetaData = mappings.get(type); + assertThat(mappingMetaData, notNullValue()); + + Map mappingSource = mappingMetaData.getSourceAsMap(); + assertFalse(mappingSource.isEmpty()); + assertTrue(mappingSource.containsKey("properties")); + + for (String fieldName : fieldNames) { + Map mappingProperties = (Map) mappingSource.get("properties"); + if (fieldName.indexOf('.') != -1) { + fieldName = fieldName.replace(".", ".properties."); + } + assertThat("field " + fieldName + " doesn't exists in mapping " + mappingMetaData.source().string(), + XContentMapValues.extractValue(fieldName, mappingProperties), notNullValue()); + } + } } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalTopHitsTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalTopHitsTests.java index c098e911c35..3da9ea7bde2 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalTopHitsTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalTopHitsTests.java @@ -28,16 +28,22 @@ import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.TopFieldDocs; import org.apache.lucene.search.TotalHits; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.document.DocumentField; import org.elasticsearch.common.io.stream.Writeable.Reader; import org.elasticsearch.common.lucene.search.TopDocsAndMaxScore; import org.elasticsearch.common.text.Text; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.aggregations.ParsedAggregation; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; import org.elasticsearch.test.InternalAggregationTestCase; +import org.elasticsearch.test.NotEqualMessageBuilder; import java.io.IOException; import java.util.ArrayList; @@ -47,6 +53,7 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import static java.lang.Math.max; @@ -209,6 +216,38 @@ public class InternalTopHitsTests extends InternalAggregationTestCase void assertEqualsWithErrorMessageFromXContent(T expected, T actual) { + if (Objects.equals(expected, actual)) { + return; + } + if (expected == null) { + throw new AssertionError("Expected null be actual was [" + actual.toString() + "]"); + } + if (actual == null) { + throw new AssertionError("Didn't expect null but actual was [null]"); + } + try (XContentBuilder actualJson = JsonXContent.contentBuilder(); + XContentBuilder expectedJson = JsonXContent.contentBuilder()) { + actualJson.startObject(); + actual.toXContent(actualJson, ToXContent.EMPTY_PARAMS); + actualJson.endObject(); + expectedJson.startObject(); + expected.toXContent(expectedJson, ToXContent.EMPTY_PARAMS); + expectedJson.endObject(); + NotEqualMessageBuilder message = new NotEqualMessageBuilder(); + message.compareMaps( + XContentHelper.convertToMap(BytesReference.bytes(actualJson), false).v2(), + XContentHelper.convertToMap(BytesReference.bytes(expectedJson), false).v2()); + throw new AssertionError("Didn't match expected value:\n" + message); + } catch (IOException e) { + throw new AssertionError("IOException while building failure message", e); + } + } + @Override protected Reader instanceReader() { return InternalTopHits::new; diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java index 3550b6aab3f..a36d75b0dd2 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java @@ -105,22 +105,6 @@ public abstract class AggregatorTestCase extends ESTestCase { private List releasables = new ArrayList<>(); private static final String TYPE_NAME = "type"; - protected AggregatorFactory createAggregatorFactory(AggregationBuilder aggregationBuilder, - IndexSearcher indexSearcher, - MappedFieldType... fieldTypes) throws IOException { - return createAggregatorFactory(aggregationBuilder, indexSearcher, createIndexSettings(), - new MultiBucketConsumer(DEFAULT_MAX_BUCKETS), fieldTypes); - } - - - protected AggregatorFactory createAggregatorFactory(AggregationBuilder aggregationBuilder, - IndexSearcher indexSearcher, - IndexSettings indexSettings, - MultiBucketConsumer bucketConsumer, - MappedFieldType... fieldTypes) throws IOException { - return createAggregatorFactory(null, aggregationBuilder, indexSearcher, indexSettings, bucketConsumer, fieldTypes); - } - /** Create a factory for the given aggregation builder. */ protected AggregatorFactory createAggregatorFactory(Query query, AggregationBuilder aggregationBuilder, @@ -164,8 +148,8 @@ public abstract class AggregatorTestCase extends ESTestCase { .collect(Collectors.toMap(MappedFieldType::name, Function.identity()))); fieldNameToType.putAll(getFieldAliases(fieldTypes)); - registerFieldTypes(queryShardContext, searchContext, mapperService, - circuitBreakerService, fieldNameToType); + registerFieldTypes(searchContext, mapperService, + fieldNameToType); return aggregationBuilder.build(searchContext, null); } @@ -178,11 +162,8 @@ public abstract class AggregatorTestCase extends ESTestCase { return Collections.emptyMap(); } - private void registerFieldTypes(QueryShardContext queryShardContext, - SearchContext searchContext, - MapperService mapperService, - CircuitBreakerService circuitBreakerService, - Map fieldNameToType) { + private static void registerFieldTypes(SearchContext searchContext, MapperService mapperService, + Map fieldNameToType) { for (Map.Entry entry : fieldNameToType.entrySet()) { String fieldName = entry.getKey(); MappedFieldType fieldType = entry.getValue(); diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractBroadcastResponseTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractBroadcastResponseTestCase.java index 87084577baa..b9eed5358e1 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/AbstractBroadcastResponseTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractBroadcastResponseTestCase.java @@ -30,7 +30,6 @@ import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.Index; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.rest.RestStatus; -import org.elasticsearch.test.AbstractXContentTestCase; import java.io.IOException; import java.util.ArrayList; diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java index bd21a19ea21..863a1deecf1 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java @@ -68,6 +68,9 @@ import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.plugins.ScriptPlugin; import org.elasticsearch.plugins.SearchPlugin; +import org.elasticsearch.script.MockScriptEngine; +import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.script.ScriptEngine; import org.elasticsearch.script.ScriptModule; import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.SearchModule; @@ -95,6 +98,7 @@ import java.util.stream.Stream; import static java.util.Collections.emptyList; import static java.util.Collections.emptyMap; +import static java.util.Collections.singletonList; import static java.util.stream.Collectors.toList; public abstract class AbstractBuilderTestCase extends ESTestCase { @@ -426,7 +430,12 @@ public abstract class AbstractBuilderTestCase extends ESTestCase { ScriptModule createScriptModule(List scriptPlugins) { if (scriptPlugins == null || scriptPlugins.isEmpty()) { - return newTestScriptModule(); + return new ScriptModule(Settings.EMPTY, singletonList(new ScriptPlugin() { + @Override + public ScriptEngine getScriptEngine(Settings settings, Collection> contexts) { + return new MockScriptEngine(MockScriptEngine.NAME, Collections.singletonMap("1", script -> "1"), emptyMap()); + } + })); } return new ScriptModule(Settings.EMPTY, scriptPlugins); } diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java index 675c5c62c4e..3da4701d9b0 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java @@ -42,7 +42,6 @@ import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRespon import org.elasticsearch.action.admin.indices.flush.FlushResponse; import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeResponse; import org.elasticsearch.action.admin.indices.get.GetIndexResponse; -import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.elasticsearch.action.admin.indices.refresh.RefreshResponse; import org.elasticsearch.action.admin.indices.segments.IndexSegments; import org.elasticsearch.action.admin.indices.segments.IndexShardSegments; @@ -71,7 +70,6 @@ import org.elasticsearch.cluster.SnapshotsInProgress; import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.cluster.metadata.IndexGraveyard; import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.cluster.metadata.MappingMetaData; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.metadata.RepositoriesMetaData; import org.elasticsearch.cluster.routing.IndexRoutingTable; @@ -84,7 +82,6 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Priority; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.network.NetworkAddress; @@ -100,11 +97,8 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.common.xcontent.NamedXContentRegistry; -import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.common.xcontent.json.JsonXContent; -import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.discovery.Discovery; import org.elasticsearch.discovery.zen.ElectMasterService; @@ -113,21 +107,17 @@ import org.elasticsearch.env.Environment; import org.elasticsearch.env.TestEnvironment; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexModule; -import org.elasticsearch.index.IndexService; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.MergePolicyConfig; import org.elasticsearch.index.MergeSchedulerConfig; import org.elasticsearch.index.MockEngineFactoryPlugin; import org.elasticsearch.index.codec.CodecService; import org.elasticsearch.index.engine.Segment; -import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MockFieldFilterPlugin; import org.elasticsearch.index.store.Store; import org.elasticsearch.index.translog.Translog; import org.elasticsearch.indices.IndicesQueryCache; import org.elasticsearch.indices.IndicesRequestCache; -import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.store.IndicesStore; import org.elasticsearch.ingest.IngestMetadata; import org.elasticsearch.node.NodeMocksPlugin; @@ -208,7 +198,6 @@ import static org.hamcrest.Matchers.emptyIterable; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.lessThanOrEqualTo; -import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.startsWith; /** @@ -404,7 +393,7 @@ public abstract class ESIntegTestCase extends ESTestCase { * Creates a randomized index template. This template is used to pass in randomized settings on a * per index basis. Allows to enable/disable the randomization for number of shards and replicas */ - public void randomIndexTemplate() { + private void randomIndexTemplate() { // TODO move settings for random directory etc here into the index based randomized settings. if (cluster().size() > 0) { @@ -513,12 +502,7 @@ public abstract class ESIntegTestCase extends ESTestCase { } private TestCluster buildWithPrivateContext(final Scope scope, final long seed) throws Exception { - return RandomizedContext.current().runWithPrivateRandomness(seed, new Callable() { - @Override - public TestCluster call() throws Exception { - return buildTestCluster(scope, seed); - } - }); + return RandomizedContext.current().runWithPrivateRandomness(seed, () -> buildTestCluster(scope, seed)); } private TestCluster buildAndPutCluster(Scope currentClusterScope, long seed) throws Exception { @@ -552,11 +536,13 @@ public abstract class ESIntegTestCase extends ESTestCase { } } - protected final void afterInternal(boolean afterClass) throws Exception { + private void afterInternal(boolean afterClass) throws Exception { boolean success = false; try { final Scope currentClusterScope = getCurrentClusterScope(); - clearDisruptionScheme(); + if (isInternalCluster()) { + internalCluster().clearDisruptionScheme(); + } try { if (cluster() != null) { if (currentClusterScope != Scope.TEST) { @@ -699,12 +685,6 @@ public abstract class ESIntegTestCase extends ESTestCase { internalCluster().setDisruptionScheme(scheme); } - public void clearDisruptionScheme() { - if (isInternalCluster()) { - internalCluster().clearDisruptionScheme(); - } - } - /** * Returns a settings object used in {@link #createIndex(String...)} and {@link #prepareCreate(String)} and friends. * This method can be overwritten by subclasses to set defaults for the indices that are created by the test. @@ -834,50 +814,6 @@ public abstract class ESIntegTestCase extends ESTestCase { assertNoTimeout(client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).get()); } - /** - * Waits until mappings for the provided fields exist on all nodes. Note, this waits for the current - * started shards and checks for concrete mappings. - */ - public void assertConcreteMappingsOnAll(final String index, final String type, final String... fieldNames) throws Exception { - Set nodes = internalCluster().nodesInclude(index); - assertThat(nodes, Matchers.not(Matchers.emptyIterable())); - for (String node : nodes) { - IndicesService indicesService = internalCluster().getInstance(IndicesService.class, node); - IndexService indexService = indicesService.indexService(resolveIndex(index)); - assertThat("index service doesn't exists on " + node, indexService, notNullValue()); - MapperService mapperService = indexService.mapperService(); - for (String fieldName : fieldNames) { - MappedFieldType fieldType = mapperService.fullName(fieldName); - assertNotNull("field " + fieldName + " doesn't exists on " + node, fieldType); - } - } - assertMappingOnMaster(index, type, fieldNames); - } - - /** - * Waits for the given mapping type to exists on the master node. - */ - public void assertMappingOnMaster(final String index, final String type, final String... fieldNames) throws Exception { - GetMappingsResponse response = client().admin().indices().prepareGetMappings(index).setTypes(type).get(); - ImmutableOpenMap mappings = response.getMappings().get(index); - assertThat(mappings, notNullValue()); - MappingMetaData mappingMetaData = mappings.get(type); - assertThat(mappingMetaData, notNullValue()); - - Map mappingSource = mappingMetaData.getSourceAsMap(); - assertFalse(mappingSource.isEmpty()); - assertTrue(mappingSource.containsKey("properties")); - - for (String fieldName : fieldNames) { - Map mappingProperties = (Map) mappingSource.get("properties"); - if (fieldName.indexOf('.') != -1) { - fieldName = fieldName.replace(".", ".properties."); - } - assertThat("field " + fieldName + " doesn't exists in mapping " + mappingMetaData.source().string(), - XContentMapValues.extractValue(fieldName, mappingProperties), notNullValue()); - } - } - /** Ensures the result counts are as expected, and logs the results if different */ public void assertResultsAndLogOnFailure(long expectedResults, SearchResponse searchResponse) { final TotalHits totalHits = searchResponse.getHits().getTotalHits(); @@ -1013,16 +949,6 @@ public abstract class ESIntegTestCase extends ESTestCase { return actionGet.getStatus(); } - /** - * Waits until at least a give number of document is visible for searchers - * - * @param numDocs number of documents to wait for. - * @return the actual number of docs seen. - */ - public long waitForDocs(final long numDocs) throws InterruptedException { - return waitForDocs(numDocs, null); - } - /** * Waits until at least a give number of document is visible for searchers * @@ -1098,22 +1024,6 @@ public abstract class ESIntegTestCase extends ESTestCase { client().admin().cluster().prepareState().get().getState(), client().admin().cluster().preparePendingClusterTasks().get()); } - /** - * Prints the segments info for the given indices as debug logging. - */ - public void logSegmentsState(String... indices) throws Exception { - IndicesSegmentResponse segsRsp = client().admin().indices().prepareSegments(indices).get(); - logger.debug("segments {} state: \n{}", indices.length == 0 ? "[_all]" : indices, - Strings.toString(segsRsp.toXContent(JsonXContent.contentBuilder().prettyPrint(), ToXContent.EMPTY_PARAMS))); - } - - /** - * Prints current memory stats as info logging. - */ - public void logMemoryStats() { - logger.info("memory: {}", Strings.toString(client().admin().cluster().prepareNodesStats().clear().setJvm(true).get(), true, true)); - } - protected void ensureClusterSizeConsistency() { if (cluster() != null && cluster().size() > 0) { // if static init fails the cluster can be null logger.trace("Check consistency for [{}] nodes", cluster().size()); @@ -1558,7 +1468,7 @@ public abstract class ESIntegTestCase extends ESTestCase { } } - private AtomicInteger dummmyDocIdGenerator = new AtomicInteger(); + private final AtomicInteger dummmyDocIdGenerator = new AtomicInteger(); /** Disables an index block for the specified index */ public static void disableIndexBlock(String index, String block) { @@ -1927,7 +1837,7 @@ public abstract class ESIntegTestCase extends ESTestCase { nodePrefix, mockPlugins, getClientWrapper(), forbidPrivateIndexSettings()); } - protected NodeConfigurationSource getNodeConfigSource() { + private NodeConfigurationSource getNodeConfigSource() { Settings.Builder initialNodeSettings = Settings.builder(); Settings.Builder initialTransportClientSettings = Settings.builder(); if (addMockTransportService()) { @@ -2280,13 +2190,13 @@ public abstract class ESIntegTestCase extends ESTestCase { */ protected static synchronized RestClient getRestClient() { if (restClient == null) { - restClient = createRestClient(null); + restClient = createRestClient(); } return restClient; } - protected static RestClient createRestClient(RestClientBuilder.HttpClientConfigCallback httpClientConfigCallback) { - return createRestClient(httpClientConfigCallback, "http"); + protected static RestClient createRestClient() { + return createRestClient(null, "http"); } protected static RestClient createRestClient(RestClientBuilder.HttpClientConfigCallback httpClientConfigCallback, String protocol) { diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java index af340f20ac6..b97d9f2b429 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java @@ -54,7 +54,6 @@ import org.elasticsearch.cluster.ClusterModule; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodeRole; -import org.elasticsearch.common.CheckedBiFunction; import org.elasticsearch.common.CheckedRunnable; import org.elasticsearch.common.SuppressForbidden; import org.elasticsearch.common.bytes.BytesReference; @@ -88,7 +87,6 @@ import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentParser.Token; import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.env.Environment; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.env.TestEnvironment; @@ -99,20 +97,11 @@ import org.elasticsearch.index.analysis.CharFilterFactory; import org.elasticsearch.index.analysis.IndexAnalyzers; import org.elasticsearch.index.analysis.TokenFilterFactory; import org.elasticsearch.index.analysis.TokenizerFactory; -import org.elasticsearch.index.mapper.Mapper; -import org.elasticsearch.index.mapper.MetadataFieldMapper; -import org.elasticsearch.indices.IndicesModule; -import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.analysis.AnalysisModule; import org.elasticsearch.plugins.AnalysisPlugin; -import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.plugins.ScriptPlugin; import org.elasticsearch.script.MockScriptEngine; import org.elasticsearch.script.Script; -import org.elasticsearch.script.ScriptContext; -import org.elasticsearch.script.ScriptEngine; -import org.elasticsearch.script.ScriptModule; import org.elasticsearch.script.ScriptType; import org.elasticsearch.search.MockSearchService; import org.elasticsearch.test.junit.listeners.LoggingListener; @@ -130,9 +119,6 @@ import org.junit.rules.RuleChain; import java.io.IOException; import java.io.InputStream; -import java.io.UncheckedIOException; -import java.nio.file.DirectoryStream; -import java.nio.file.Files; import java.nio.file.Path; import java.security.Security; import java.time.ZoneId; @@ -161,7 +147,6 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import static java.util.Collections.emptyMap; -import static java.util.Collections.singletonList; import static org.elasticsearch.common.util.CollectionUtils.arrayAsArrayList; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; @@ -971,10 +956,6 @@ public abstract class ESTestCase extends LuceneTestCase { } } - public Path getBwcIndicesPath() { - return getDataPath("/indices/bwc"); - } - /** Returns a random number of temporary paths. */ public String[] tmpPaths() { final int numPaths = TestUtil.nextInt(random(), 1, 3); @@ -1081,21 +1062,8 @@ public abstract class ESTestCase extends LuceneTestCase { */ protected final BytesReference toShuffledXContent(ToXContent toXContent, XContentType xContentType, ToXContent.Params params, boolean humanReadable, String... exceptFieldNames) throws IOException{ - return toShuffledXContent(toXContent, xContentType, params, humanReadable, this::createParser, exceptFieldNames); - } - - /** - * Returns the bytes that represent the XContent output of the provided {@link ToXContent} object, using the provided - * {@link XContentType}. Wraps the output into a new anonymous object according to the value returned - * by the {@link ToXContent#isFragment()} method returns. Shuffles the keys to make sure that parsing never relies on keys ordering. - */ - protected static BytesReference toShuffledXContent(ToXContent toXContent, XContentType xContentType, ToXContent.Params params, - boolean humanReadable, - CheckedBiFunction - parserFunction, - String... exceptFieldNames) throws IOException{ BytesReference bytes = XContentHelper.toXContent(toXContent, xContentType, params, humanReadable); - try (XContentParser parser = parserFunction.apply(xContentType.xContent(), bytes)) { + try (XContentParser parser = createParser(xContentType.xContent(), bytes)) { try (XContentBuilder builder = shuffleXContent(parser, rarely(), exceptFieldNames)) { return BytesReference.bytes(builder); } @@ -1217,77 +1185,6 @@ public abstract class ESTestCase extends LuceneTestCase { } } - public void assertAllIndicesRemovedAndDeletionCompleted(Iterable indicesServices) throws Exception { - for (IndicesService indicesService : indicesServices) { - assertBusy(() -> assertFalse(indicesService.iterator().hasNext()), 1, TimeUnit.MINUTES); - assertBusy(() -> assertFalse(indicesService.hasUncompletedPendingDeletes()), 1, TimeUnit.MINUTES); - } - } - - /** - * Asserts that there are no files in the specified path - */ - public void assertPathHasBeenCleared(Path path) { - logger.info("--> checking that [{}] has been cleared", path); - int count = 0; - StringBuilder sb = new StringBuilder(); - sb.append("["); - if (Files.exists(path)) { - try (DirectoryStream stream = Files.newDirectoryStream(path)) { - for (Path file : stream) { - // Skip files added by Lucene's ExtraFS - if (file.getFileName().toString().startsWith("extra")) { - continue; - } - logger.info("--> found file: [{}]", file.toAbsolutePath().toString()); - if (Files.isDirectory(file)) { - assertPathHasBeenCleared(file); - } else if (Files.isRegularFile(file)) { - count++; - sb.append(file.toAbsolutePath().toString()); - sb.append("\n"); - } - } - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - sb.append("]"); - assertThat(count + " files exist that should have been cleaned:\n" + sb.toString(), count, equalTo(0)); - } - - /** - * Assert that two objects are equals, calling {@link ToXContent#toXContent(XContentBuilder, ToXContent.Params)} to print out their - * differences if they aren't equal. - */ - public static void assertEqualsWithErrorMessageFromXContent(T expected, T actual) { - if (Objects.equals(expected, actual)) { - return; - } - if (expected == null) { - throw new AssertionError("Expected null be actual was [" + actual.toString() + "]"); - } - if (actual == null) { - throw new AssertionError("Didn't expect null but actual was [null]"); - } - try (XContentBuilder actualJson = JsonXContent.contentBuilder(); - XContentBuilder expectedJson = JsonXContent.contentBuilder()) { - actualJson.startObject(); - actual.toXContent(actualJson, ToXContent.EMPTY_PARAMS); - actualJson.endObject(); - expectedJson.startObject(); - expected.toXContent(expectedJson, ToXContent.EMPTY_PARAMS); - expectedJson.endObject(); - NotEqualMessageBuilder message = new NotEqualMessageBuilder(); - message.compareMaps( - XContentHelper.convertToMap(BytesReference.bytes(actualJson), false).v2(), - XContentHelper.convertToMap(BytesReference.bytes(expectedJson), false).v2()); - throw new AssertionError("Didn't match expected value:\n" + message); - } catch (IOException e) { - throw new AssertionError("IOException while building failure message", e); - } - } - /** * Create a new {@link XContentParser}. */ @@ -1342,7 +1239,7 @@ public abstract class ESTestCase extends LuceneTestCase { * Create a "mock" script for use either with {@link MockScriptEngine} or anywhere where you need a script but don't really care about * its contents. */ - public static final Script mockScript(String id) { + public static Script mockScript(String id) { return new Script(ScriptType.INLINE, MockScriptEngine.NAME, id, emptyMap()); } @@ -1420,32 +1317,6 @@ public abstract class ESTestCase extends LuceneTestCase { analysisRegistry.buildCharFilterFactories(indexSettings)); } - public static ScriptModule newTestScriptModule() { - return new ScriptModule(Settings.EMPTY, singletonList(new ScriptPlugin() { - @Override - public ScriptEngine getScriptEngine(Settings settings, Collection> contexts) { - return new MockScriptEngine(MockScriptEngine.NAME, Collections.singletonMap("1", script -> "1"), Collections.emptyMap()); - } - })); - } - - /** Creates an IndicesModule for testing with the given mappers and metadata mappers. */ - public static IndicesModule newTestIndicesModule(Map extraMappers, - Map extraMetadataMappers) { - return new IndicesModule(Collections.singletonList( - new MapperPlugin() { - @Override - public Map getMappers() { - return extraMappers; - } - @Override - public Map getMetadataMappers() { - return extraMetadataMappers; - } - } - )); - } - /** * This cute helper class just holds all analysis building blocks that are used * to build IndexAnalyzers. This is only for testing since in production we only need the diff --git a/test/framework/src/main/java/org/elasticsearch/test/FieldMaskingReader.java b/test/framework/src/main/java/org/elasticsearch/test/FieldMaskingReader.java index 3aba68868f7..78bb3d9287f 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/FieldMaskingReader.java +++ b/test/framework/src/main/java/org/elasticsearch/test/FieldMaskingReader.java @@ -23,7 +23,6 @@ import org.apache.lucene.index.FieldFilterLeafReader; import org.apache.lucene.index.FilterDirectoryReader; import org.apache.lucene.index.FilterLeafReader; import org.apache.lucene.index.LeafReader; -import org.apache.lucene.index.IndexReader.CacheHelper; import java.io.IOException; import java.util.Collections; @@ -67,4 +66,4 @@ public class FieldMaskingReader extends FilterDirectoryReader { public CacheHelper getReaderCacheHelper() { return in.getReaderCacheHelper(); } -} \ No newline at end of file +} diff --git a/test/framework/src/main/java/org/elasticsearch/test/engine/MockEngineSupport.java b/test/framework/src/main/java/org/elasticsearch/test/engine/MockEngineSupport.java index 7980865a3f8..21cb4ca652f 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/engine/MockEngineSupport.java +++ b/test/framework/src/main/java/org/elasticsearch/test/engine/MockEngineSupport.java @@ -24,8 +24,6 @@ import org.apache.lucene.index.AssertingDirectoryReader; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.FilterDirectoryReader; import org.apache.lucene.index.IndexReader; -import org.apache.lucene.search.QueryCache; -import org.apache.lucene.search.QueryCachingPolicy; import org.apache.lucene.util.LuceneTestCase; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.settings.Setting; @@ -67,8 +65,6 @@ public final class MockEngineSupport { private final AtomicBoolean closing = new AtomicBoolean(false); private final Logger logger = LogManager.getLogger(Engine.class); private final ShardId shardId; - private final QueryCache filterCache; - private final QueryCachingPolicy filterCachingPolicy; private final InFlightSearchers inFlightSearchers; private final MockContext mockContext; private final boolean disableFlushOnClose; @@ -95,8 +91,6 @@ public final class MockEngineSupport { public MockEngineSupport(EngineConfig config, Class wrapper) { Settings settings = config.getIndexSettings().getSettings(); shardId = config.getShardId(); - filterCache = config.getQueryCache(); - filterCachingPolicy = config.getQueryCachingPolicy(); final long seed = config.getIndexSettings().getValue(ESIntegTestCase.INDEX_TEST_SEED_SETTING); Random random = new Random(seed); final double ratio = WRAP_READER_RATIO.get(settings);