honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[TEST] Random dynamic templates. 

This change randomly indexes the _id field and randomizes field data formats
and loading.

Close #5834
This commit is contained in:
Adrien Grand 2014-05-06 10:01:15 +02:00
parent 76463ee2b2
commit 17a32fca03
11 changed files with 138 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/main/java/org/elasticsearch/search/suggest/context/GeolocationContextMapping.java
View File

@ -621,9 +621,13 @@ public class GeolocationContextMapping extends ContextMapping {
for (int i = 0 ; i < lonFields.length ; i++) {
IndexableField lonField = lonFields[i];
IndexableField latField = latFields[i];
assert lonField.fieldType().docValueType() == latField.fieldType().docValueType();
// we write doc values fields differently: one field for all values, so we need to only care about indexed fields
if (lonField.fieldType().docValueType() == null) {
spare.reset(latField.numericValue().doubleValue(), lonField.numericValue().doubleValue());
geohashes.add(spare.geohash());
}
}
} else {
geohashes = mapping.defaultLocations;
}

2
src/test/java/org/elasticsearch/index/fielddata/DisabledFieldDataFormatTests.java
View File

@ -24,10 +24,12 @@ import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.test.ElasticsearchIntegrationTest;
import org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertFailures;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
@ClusterScope(randomDynamicTemplates = false)
public class DisabledFieldDataFormatTests extends ElasticsearchIntegrationTest {
@Override

4
src/test/java/org/elasticsearch/indices/cache/CacheTests.java vendored
View File

@ -29,16 +29,16 @@ import org.elasticsearch.index.query.FilterBuilders;
import org.elasticsearch.search.sort.SortOrder;
import org.elasticsearch.test.ElasticsearchIntegrationTest;
import org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope;
import org.elasticsearch.test.ElasticsearchIntegrationTest.Scope;
import org.junit.Test;
import static org.elasticsearch.index.query.QueryBuilders.filteredQuery;
import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
import static org.elasticsearch.test.ElasticsearchIntegrationTest.*;
import static org.hamcrest.Matchers.*;
/**
*/
@ClusterScope(scope= Scope.SUITE, numDataNodes =1, numClientNodes = 0)
@ClusterScope(scope= Scope.SUITE, numDataNodes =1, numClientNodes = 0, randomDynamicTemplates = false)
public class CacheTests extends ElasticsearchIntegrationTest {
@Override

4
src/test/java/org/elasticsearch/indices/fielddata/breaker/CircuitBreakerServiceTests.java
View File

@ -24,6 +24,7 @@ import org.elasticsearch.common.collect.MapBuilder;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.rest.RestStatus;
import org.elasticsearch.test.ElasticsearchIntegrationTest;
import org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope;
import org.elasticsearch.test.junit.annotations.TestLogging;
import org.junit.Test;
@ -31,7 +32,6 @@ import java.util.Arrays;
import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS;
import static org.elasticsearch.common.settings.ImmutableSettings.settingsBuilder;
import static org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope;
import static org.elasticsearch.test.ElasticsearchIntegrationTest.Scope.TEST;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertFailures;
@ -40,7 +40,7 @@ import static org.hamcrest.CoreMatchers.containsString;
/**
* Integration tests for InternalCircuitBreakerService
*/
@ClusterScope(scope = TEST)
@ClusterScope(scope = TEST, randomDynamicTemplates = false)
public class CircuitBreakerServiceTests extends ElasticsearchIntegrationTest {
private String randomRidiculouslySmallLimit() {

2
src/test/java/org/elasticsearch/indices/mapping/SimpleGetMappingsTests.java
View File

@ -24,6 +24,7 @@ import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
import org.elasticsearch.common.Priority;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.test.ElasticsearchIntegrationTest;
import org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope;
import org.junit.Test;
import java.io.IOException;
@ -35,6 +36,7 @@ import static org.hamcrest.Matchers.notNullValue;
/**
*
*/
@ClusterScope(randomDynamicTemplates = false)
public class SimpleGetMappingsTests extends ElasticsearchIntegrationTest {
@Test

2
src/test/java/org/elasticsearch/indices/mapping/UpdateMappingTests.java
View File

@ -40,6 +40,7 @@ import org.elasticsearch.index.mapper.MapperParsingException;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.mapper.MergeMappingException;
import org.elasticsearch.test.ElasticsearchIntegrationTest;
import org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope;
import org.hamcrest.Matchers;
import org.junit.Test;
@ -55,6 +56,7 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcke
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertThrows;
import static org.hamcrest.Matchers.*;
@ClusterScope(randomDynamicTemplates = false)
public class UpdateMappingTests extends ElasticsearchIntegrationTest {
@Test

6
src/test/java/org/elasticsearch/routing/SimpleRoutingTests.java
View File

@ -303,6 +303,12 @@ public class SimpleRoutingTests extends ElasticsearchIntegrationTest {
client().admin().indices().prepareCreate("test")
.addMapping("type1", XContentFactory.jsonBuilder().startObject().startObject("type1")
.startObject("_routing").field("required", true).field("path", "routing_field").endObject()
.startObject("properties")
.startObject("routing_field")
.field("type", "long")
.field("doc_values", false) // TODO this test fails with doc values https://github.com/elasticsearch/elasticsearch/pull/5858
.endObject()
.endObject()
.endObject().endObject())
.execute().actionGet();
ensureGreen();

30
src/test/java/org/elasticsearch/search/child/SimpleChildQuerySearchTests.java
View File

@ -33,6 +33,9 @@ import org.elasticsearch.action.search.SearchType;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.common.lucene.search.function.CombineFunction;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.fielddata.FieldDataType;
import org.elasticsearch.index.mapper.FieldMapper.Loading;
import org.elasticsearch.index.mapper.MergeMappingException;
import org.elasticsearch.index.query.*;
import org.elasticsearch.index.search.child.ScoreType;
@ -241,8 +244,18 @@ public class SimpleChildQuerySearchTests extends ElasticsearchIntegrationTest {
@Test
public void testClearIdCacheBug() throws Exception {
// enforce lazy loading to make sure that p/c stats are not counted as part of field data
assertAcked(prepareCreate("test")
.addMapping("parent"));
.addMapping("parent", XContentFactory.jsonBuilder().startObject().startObject("parent")
.startObject("properties")
.startObject("p_field")
.field("type", "string")
.startObject("fielddata")
.field(FieldDataType.FORMAT_KEY, Loading.LAZY)
.endObject()
.endObject()
.endObject().endObject().endObject()));
ensureGreen();
client().prepareIndex("test", "parent", "p0").setSource("p_field", "p_value0").get();
@ -256,7 +269,18 @@ public class SimpleChildQuerySearchTests extends ElasticsearchIntegrationTest {
// Now add mapping + children
client().admin().indices().preparePutMapping("test").setType("child")
.setSource("_parent", "type=parent")
.setSource(XContentFactory.jsonBuilder().startObject().startObject("child")
.startObject("_parent")
.field("type", "parent")
.endObject()
.startObject("properties")
.startObject("c_field")
.field("type", "string")
.startObject("fielddata")
.field(FieldDataType.FORMAT_KEY, Loading.LAZY)
.endObject()
.endObject()
.endObject().endObject().endObject())
.get();
// index simple data
@ -1460,7 +1484,7 @@ public class SimpleChildQuerySearchTests extends ElasticsearchIntegrationTest {
GetMappingsResponse getMappingsResponse = client().admin().indices().prepareGetMappings("test").get();
Map<String, Object> mapping = getMappingsResponse.getMappings().get("test").get("child").getSourceAsMap();
assertThat(mapping.size(), equalTo(1));
assertThat(mapping.size(), greaterThanOrEqualTo(1)); // there are potentially some meta fields configured randomly
assertThat(mapping.get("properties"), notNullValue());
try {

84
src/test/java/org/elasticsearch/test/ElasticsearchIntegrationTest.java
View File

@ -39,6 +39,7 @@ import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRespon
import org.elasticsearch.action.admin.indices.flush.FlushResponse;
import org.elasticsearch.action.admin.indices.optimize.OptimizeResponse;
import org.elasticsearch.action.admin.indices.refresh.RefreshResponse;
import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequestBuilder;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.get.GetResponse;
@ -65,8 +66,12 @@ import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.discovery.zen.elect.ElectMasterService;
import org.elasticsearch.index.fielddata.FieldDataType;
import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.index.mapper.FieldMapper.Loading;
import org.elasticsearch.index.mapper.internal.IdFieldMapper;
import org.elasticsearch.index.merge.policy.*;
import org.elasticsearch.index.merge.scheduler.ConcurrentMergeSchedulerProvider;
import org.elasticsearch.index.merge.scheduler.MergeSchedulerModule;
@ -294,7 +299,7 @@ public abstract class ElasticsearchIntegrationTest extends ElasticsearchTestCase
* Creates a randomized index template. This template is used to pass in randomized settings on a
* per index basis. Allows to enable/disable the randomization for number of shards and replicas
*/
private void randomIndexTemplate() {
private void randomIndexTemplate() throws IOException {
// TODO move settings for random directory etc here into the index based randomized settings.
if (immutableCluster().size() > 0) {
ImmutableSettings.Builder randomSettingsBuilder =
@ -306,11 +311,72 @@ public abstract class ElasticsearchIntegrationTest extends ElasticsearchTestCase
//use either 0 or 1 replica, yet a higher amount when possible, but only rarely
.put(SETTING_NUMBER_OF_REPLICAS, between(0, getRandom().nextInt(10) > 0 ? 1 : immutableCluster().numDataNodes() - 1));
}
client().admin().indices().preparePutTemplate("random_index_template")
XContentBuilder mappings = null;
if (frequently() && randomDynamicTemplates()) {
mappings = XContentFactory.jsonBuilder().startObject().startObject("_default_");
if (randomBoolean()) {
mappings.startObject(IdFieldMapper.NAME)
.field("index", randomFrom("not_analyzed", "no"))
.endObject();
}
mappings.startArray("dynamic_templates")
.startObject()
.startObject("template-strings")
.field("match_mapping_type", "string")
.startObject("mapping")
.startObject("fielddata")
.field(FieldDataType.FORMAT_KEY, randomFrom("paged_bytes", "fst")) // unfortunately doc values only work on not_analyzed fields
.field(Loading.KEY, randomFrom(Loading.values()))
.endObject()
.endObject()
.endObject()
.endObject()
.startObject()
.startObject("template-longs")
.field("match_mapping_type", "long")
.startObject("mapping")
.startObject("fielddata")
.field(FieldDataType.FORMAT_KEY, randomFrom("array", "doc_values"))
.field(Loading.KEY, randomFrom(Loading.LAZY, Loading.EAGER))
.endObject()
.endObject()
.endObject()
.endObject()
.startObject()
.startObject("template-doubles")
.field("match_mapping_type", "double")
.startObject("mapping")
.startObject("fielddata")
.field(FieldDataType.FORMAT_KEY, randomFrom("array", "doc_values"))
.field(Loading.KEY, randomFrom(Loading.LAZY, Loading.EAGER))
.endObject()
.endObject()
.endObject()
.endObject()
.startObject()
.startObject("template-geo_points")
.field("match_mapping_type", "geo_point")
.startObject("mapping")
.startObject("fielddata")
.field(FieldDataType.FORMAT_KEY, randomFrom("array", "doc_values"))
.field(Loading.KEY, randomFrom(Loading.LAZY, Loading.EAGER))
.endObject()
.endObject()
.endObject()
.endObject()
.endArray();
mappings.endObject().endObject();
}
PutIndexTemplateRequestBuilder putTemplate = client().admin().indices()
.preparePutTemplate("random_index_template")
.setTemplate("*")
.setOrder(0)
.setSettings(randomSettingsBuilder)
.execute().actionGet();
.setSettings(randomSettingsBuilder);
if (mappings != null) {
putTemplate.addMapping("_default_", mappings);
}
assertAcked(putTemplate.execute().actionGet());
}
}
@ -1057,6 +1123,11 @@ public abstract class ElasticsearchIntegrationTest extends ElasticsearchTestCase
* ratio in the interval <code>[0..1]</code> is used.
*/
double transportClientRatio() default -1;
/**
* Return whether or not to enable dynamic templates for the mappings.
*/
boolean randomDynamicTemplates() default true;
}
private class LatchedActionListener<Response> implements ActionListener<Response> {
@ -1148,6 +1219,11 @@ public abstract class ElasticsearchIntegrationTest extends ElasticsearchTestCase
return annotation == null ? TestCluster.DEFAULT_NUM_CLIENT_NODES : annotation.numClientNodes();
}
private boolean randomDynamicTemplates() {
ClusterScope annotation = getAnnotation(this.getClass());
return annotation == null ? true : annotation.randomDynamicTemplates();
}
/**
* This method is used to obtain settings for the <tt>Nth</tt> node in the cluster.
* Nodes in this cluster are associated with an ordinal number such that nodes can

13
src/test/java/org/elasticsearch/test/ElasticsearchTestCase.java
View File

@ -18,11 +18,9 @@
*/
package org.elasticsearch.test;
import com.carrotsearch.randomizedtesting.annotations.Listeners;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope;
import com.carrotsearch.randomizedtesting.RandomizedTest;
import com.carrotsearch.randomizedtesting.annotations.*;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope.Scope;
import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;
import com.google.common.base.Predicate;
import com.google.common.collect.ImmutableList;
import org.apache.lucene.store.MockDirectoryWrapper;
@ -30,7 +28,6 @@ import org.apache.lucene.util.AbstractRandomizedTest;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TimeUnits;
import org.elasticsearch.Version;
import org.elasticsearch.test.cache.recycler.MockPageCacheRecycler;
import org.elasticsearch.client.Requests;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.logging.ESLogger;
@ -39,11 +36,11 @@ import org.elasticsearch.common.util.concurrent.EsAbortPolicy;
import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.test.cache.recycler.MockBigArrays;
import org.elasticsearch.test.cache.recycler.MockPageCacheRecycler;
import org.elasticsearch.test.junit.listeners.LoggingListener;
import org.elasticsearch.test.store.MockDirectoryHelper;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import java.io.Closeable;
@ -293,4 +290,8 @@ public abstract class ElasticsearchTestCase extends AbstractRandomizedTest {
return threadGroup.getName();
}
}
public static <T> T randomFrom(T... values) {
return RandomizedTest.randomFrom(values);
}
}

2
src/test/java/org/elasticsearch/test/rest/ElasticsearchRestTests.java
View File

@ -25,6 +25,7 @@ import com.google.common.collect.Lists;
import org.elasticsearch.Version;
import org.elasticsearch.common.Strings;
import org.elasticsearch.test.ElasticsearchIntegrationTest;
import org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope;
import org.elasticsearch.test.rest.client.RestException;
import org.elasticsearch.test.rest.parser.RestTestParseException;
import org.elasticsearch.test.rest.parser.RestTestSuiteParser;
@ -51,6 +52,7 @@ import java.util.Set;
//tests distribution disabled for now since it causes reporting problems,
// due to the non unique suite name
//@ReplicateOnEachVm
@ClusterScope(randomDynamicTemplates = false)
public class ElasticsearchRestTests extends ElasticsearchIntegrationTest {
/**