honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Index Template: parse and validate mappings in template when template puts 

Share applying template with MetaDataCreateIndexService and MetaDataIndexTemplateService
Add some unit test
Collapse addMappingsToMapperService and move it to MapperService
Extract validateTemplate method
use expectThrows in testcase
Add TODO comment

Closes #2415
This commit is contained in:
Jun Ohtani 2014-12-04 17:42:40 +09:00
parent 93c5a9dacd
commit fd76291131
8 changed files with 359 additions and 113 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java
View File

@ -55,9 +55,7 @@ import org.elasticsearch.common.io.PathUtils;
import org.elasticsearch.common.regex.Regex;
import org.elasticsearch.common.settings.IndexScopedSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.env.Environment;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.IndexNotFoundException;
@ -220,7 +218,7 @@ public class MetaDataCreateIndexService extends AbstractComponent {
List<String> templateNames = new ArrayList<>();
for (Map.Entry<String, String> entry : request.mappings().entrySet()) {
mappings.put(entry.getKey(), parseMapping(entry.getValue()));
mappings.put(entry.getKey(), MapperService.parseMapping(entry.getValue()));
}
for (Map.Entry<String, Custom> entry : request.customs().entrySet()) {
@ -232,9 +230,9 @@ public class MetaDataCreateIndexService extends AbstractComponent {
templateNames.add(template.getName());
for (ObjectObjectCursor<String, CompressedXContent> cursor : template.mappings()) {
if (mappings.containsKey(cursor.key)) {
XContentHelper.mergeDefaults(mappings.get(cursor.key), parseMapping(cursor.value.string()));
XContentHelper.mergeDefaults(mappings.get(cursor.key), MapperService.parseMapping(cursor.value.string()));
} else {
mappings.put(cursor.key, parseMapping(cursor.value.string()));
mappings.put(cursor.key, MapperService.parseMapping(cursor.value.string()));
}
}
// handle custom
@ -315,26 +313,11 @@ public class MetaDataCreateIndexService extends AbstractComponent {
createdIndex = indexService.index();
// now add the mappings
MapperService mapperService = indexService.mapperService();
// first, add the default mapping
if (mappings.containsKey(MapperService.DEFAULT_MAPPING)) {
try {
mapperService.merge(MapperService.DEFAULT_MAPPING, new CompressedXContent(XContentFactory.jsonBuilder().map(mappings.get(MapperService.DEFAULT_MAPPING)).string()), MapperService.MergeReason.MAPPING_UPDATE, request.updateAllTypes());
} catch (Exception e) {
removalReason = "failed on parsing default mapping on index creation";
throw new MapperParsingException("Failed to parse mapping [{}]: {}", e, MapperService.DEFAULT_MAPPING, e.getMessage());
}
}
for (Map.Entry<String, Map<String, Object>> entry : mappings.entrySet()) {
if (entry.getKey().equals(MapperService.DEFAULT_MAPPING)) {
continue;
}
try {
// apply the default here, its the first time we parse it
mapperService.merge(entry.getKey(), new CompressedXContent(XContentFactory.jsonBuilder().map(entry.getValue()).string()), MapperService.MergeReason.MAPPING_UPDATE, request.updateAllTypes());
} catch (Exception e) {
removalReason = "failed on parsing mappings on index creation";
throw new MapperParsingException("Failed to parse mapping [{}]: {}", e, entry.getKey(), e.getMessage());
}
try {
mapperService.merge(mappings, request.updateAllTypes());
} catch (MapperParsingException mpe) {
removalReason = "failed on parsing default mapping/mappings on index creation";
throw mpe;
}
final QueryShardContext queryShardContext = indexService.newQueryShardContext();
@ -426,12 +409,6 @@ public class MetaDataCreateIndexService extends AbstractComponent {
});
}
private Map<String, Object> parseMapping(String mappingSource) throws Exception {
try (XContentParser parser = XContentFactory.xContent(mappingSource).createParser(mappingSource)) {
return parser.map();
}
}
private List<IndexTemplateMetaData> findTemplates(CreateIndexClusterStateUpdateRequest request, ClusterState state, IndexTemplateFilter indexTemplateFilter) throws IOException {
List<IndexTemplateMetaData> templates = new ArrayList<>();
for (ObjectCursor<IndexTemplateMetaData> cursor : state.metaData().templates().values()) {

97
core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java
View File

@ -19,6 +19,7 @@
package org.elasticsearch.cluster.metadata;
import com.carrotsearch.hppc.cursors.ObjectCursor;
import org.elasticsearch.Version;
import org.elasticsearch.action.admin.indices.alias.Alias;
import org.elasticsearch.action.support.master.MasterNodeRequest;
import org.elasticsearch.cluster.ClusterState;
@ -26,17 +27,25 @@ import org.elasticsearch.cluster.ClusterStateUpdateTask;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.Priority;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.UUIDs;
import org.elasticsearch.common.ValidationException;
import org.elasticsearch.common.component.AbstractComponent;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.regex.Regex;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.NodeServicesProvider;
import org.elasticsearch.index.IndexService;
import org.elasticsearch.index.mapper.MapperParsingException;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.indices.IndexTemplateAlreadyExistsException;
import org.elasticsearch.indices.IndexTemplateMissingException;
import org.elasticsearch.indices.IndicesService;
import org.elasticsearch.indices.InvalidIndexTemplateException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@ -51,14 +60,18 @@ public class MetaDataIndexTemplateService extends AbstractComponent {
private final ClusterService clusterService;
private final AliasValidator aliasValidator;
private final IndicesService indicesService;
private final MetaDataCreateIndexService metaDataCreateIndexService;
private final NodeServicesProvider nodeServicesProvider;
@Inject
public MetaDataIndexTemplateService(Settings settings, ClusterService clusterService, MetaDataCreateIndexService metaDataCreateIndexService, AliasValidator aliasValidator) {
public MetaDataIndexTemplateService(Settings settings, ClusterService clusterService, MetaDataCreateIndexService metaDataCreateIndexService, AliasValidator aliasValidator, IndicesService indicesService, NodeServicesProvider nodeServicesProvider) {
super(settings);
this.clusterService = clusterService;
this.aliasValidator = aliasValidator;
this.indicesService = indicesService;
this.metaDataCreateIndexService = metaDataCreateIndexService;
this.nodeServicesProvider = nodeServicesProvider;
}
public void removeTemplates(final RemoveRequest request, final RemoveListener listener) {
@ -126,28 +139,7 @@ public class MetaDataIndexTemplateService extends AbstractComponent {
return;
}
IndexTemplateMetaData.Builder templateBuilder;
try {
templateBuilder = IndexTemplateMetaData.builder(request.name);
templateBuilder.order(request.order);
templateBuilder.template(request.template);
templateBuilder.settings(request.settings);
for (Map.Entry<String, String> entry : request.mappings.entrySet()) {
templateBuilder.putMapping(entry.getKey(), entry.getValue());
}
for (Alias alias : request.aliases) {
AliasMetaData aliasMetaData = AliasMetaData.builder(alias.name()).filter(alias.filter())
.indexRouting(alias.indexRouting()).searchRouting(alias.searchRouting()).build();
templateBuilder.putAlias(aliasMetaData);
}
for (Map.Entry<String, IndexMetaData.Custom> entry : request.customs.entrySet()) {
templateBuilder.putCustom(entry.getKey(), entry.getValue());
}
} catch (Throwable e) {
listener.onFailure(e);
return;
}
final IndexTemplateMetaData template = templateBuilder.build();
final IndexTemplateMetaData.Builder templateBuilder = IndexTemplateMetaData.builder(request.name);
clusterService.submitStateUpdateTask("create-index-template [" + request.name + "], cause [" + request.cause + "]",
new ClusterStateUpdateTask(Priority.URGENT) {
@ -163,10 +155,23 @@ public class MetaDataIndexTemplateService extends AbstractComponent {
}
@Override
public ClusterState execute(ClusterState currentState) {
public ClusterState execute(ClusterState currentState) throws Exception {
if (request.create && currentState.metaData().templates().containsKey(request.name)) {
throw new IndexTemplateAlreadyExistsException(request.name);
}
validateAndAddTemplate(request, templateBuilder, indicesService, nodeServicesProvider, metaDataCreateIndexService);
for (Alias alias : request.aliases) {
AliasMetaData aliasMetaData = AliasMetaData.builder(alias.name()).filter(alias.filter())
.indexRouting(alias.indexRouting()).searchRouting(alias.searchRouting()).build();
templateBuilder.putAlias(aliasMetaData);
}
for (Map.Entry<String, IndexMetaData.Custom> entry : request.customs.entrySet()) {
templateBuilder.putCustom(entry.getKey(), entry.getValue());
}
IndexTemplateMetaData template = templateBuilder.build();
MetaData.Builder builder = MetaData.builder(currentState.metaData()).put(template);
return ClusterState.builder(currentState).metaData(builder).build();
@ -174,11 +179,53 @@ public class MetaDataIndexTemplateService extends AbstractComponent {
@Override
public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
listener.onResponse(new PutResponse(true, template));
listener.onResponse(new PutResponse(true, templateBuilder.build()));
}
});
}
private static void validateAndAddTemplate(final PutRequest request, IndexTemplateMetaData.Builder templateBuilder, IndicesService indicesService,
NodeServicesProvider nodeServicesProvider, MetaDataCreateIndexService metaDataCreateIndexService) throws Exception {
Index createdIndex = null;
final String temporaryIndexName = UUIDs.randomBase64UUID();
try {
//create index service for parsing and validating "mappings"
Settings dummySettings = Settings.builder()
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
.put(request.settings)
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
.put(IndexMetaData.SETTING_INDEX_UUID, UUIDs.randomBase64UUID())
.build();
final IndexMetaData tmpIndexMetadata = IndexMetaData.builder(temporaryIndexName).settings(dummySettings).build();
IndexService dummyIndexService = indicesService.createIndex(nodeServicesProvider, tmpIndexMetadata, Collections.emptyList());
createdIndex = dummyIndexService.index();
templateBuilder.order(request.order);
templateBuilder.template(request.template);
templateBuilder.settings(request.settings);
Map<String, Map<String, Object>> mappingsForValidation = new HashMap<>();
for (Map.Entry<String, String> entry : request.mappings.entrySet()) {
try {
templateBuilder.putMapping(entry.getKey(), entry.getValue());
} catch (Exception e) {
throw new MapperParsingException("Failed to parse mapping [{}]: {}", e, entry.getKey(), e.getMessage());
}
mappingsForValidation.put(entry.getKey(), MapperService.parseMapping(entry.getValue()));
}
dummyIndexService.mapperService().merge(mappingsForValidation, false);
} finally {
if (createdIndex != null) {
indicesService.removeIndex(createdIndex, " created for parsing template mapping");
}
}
}
private void validate(PutRequest request) {
List<String> validationErrors = new ArrayList<>();
if (request.name.contains(" ")) {

31
core/src/main/java/org/elasticsearch/index/mapper/MapperService.java
View File

@ -29,6 +29,8 @@ import org.elasticsearch.common.compress.CompressedXContent;
import org.elasticsearch.common.regex.Regex;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Setting.Property;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.AbstractIndexComponent;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.analysis.AnalysisService;
@ -175,6 +177,35 @@ public class MapperService extends AbstractIndexComponent {
return this.documentParser;
}
public static Map<String, Object> parseMapping(String mappingSource) throws Exception {
try (XContentParser parser = XContentFactory.xContent(mappingSource).createParser(mappingSource)) {
return parser.map();
}
}
//TODO: make this atomic
public void merge(Map<String, Map<String, Object>> mappings, boolean updateAllTypes) throws MapperParsingException {
// first, add the default mapping
if (mappings.containsKey(DEFAULT_MAPPING)) {
try {
this.merge(DEFAULT_MAPPING, new CompressedXContent(XContentFactory.jsonBuilder().map(mappings.get(DEFAULT_MAPPING)).string()), MergeReason.MAPPING_UPDATE, updateAllTypes);
} catch (Exception e) {
throw new MapperParsingException("Failed to parse mapping [{}]: {}", e, DEFAULT_MAPPING, e.getMessage());
}
}
for (Map.Entry<String, Map<String, Object>> entry : mappings.entrySet()) {
if (entry.getKey().equals(DEFAULT_MAPPING)) {
continue;
}
try {
// apply the default here, its the first time we parse it
this.merge(entry.getKey(), new CompressedXContent(XContentFactory.jsonBuilder().map(entry.getValue()).string()), MergeReason.MAPPING_UPDATE, updateAllTypes);
} catch (Exception e) {
throw new MapperParsingException("Failed to parse mapping [{}]: {}", e, entry.getKey(), e.getMessage());
}
}
}
public DocumentMapper merge(String type, CompressedXContent mappingSource, MergeReason reason, boolean updateAllTypes) {
if (DEFAULT_MAPPING.equals(type)) {
// verify we can parse it

110
core/src/test/java/org/elasticsearch/action/admin/indices/template/put/MetaDataIndexTemplateServiceTests.java
View File

@ -26,9 +26,14 @@ import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.metadata.MetaDataCreateIndexService;
import org.elasticsearch.cluster.metadata.MetaDataIndexTemplateService;
import org.elasticsearch.cluster.metadata.MetaDataIndexTemplateService.PutRequest;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.NodeServicesProvider;
import org.elasticsearch.index.mapper.MapperParsingException;
import org.elasticsearch.indices.IndicesService;
import org.elasticsearch.indices.InvalidIndexTemplateException;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.test.ESSingleNodeTestCase;
import java.util.ArrayList;
import java.util.Collections;
@ -36,12 +41,14 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.CountDownLatch;
import static org.hamcrest.CoreMatchers.containsString;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.CoreMatchers.instanceOf;
public class MetaDataIndexTemplateServiceTests extends ESTestCase {
public class MetaDataIndexTemplateServiceTests extends ESSingleNodeTestCase {
public void testIndexTemplateInvalidNumberOfShards() {
PutRequest request = new PutRequest("test", "test_shards");
request.template("test_shards*");
@ -83,6 +90,68 @@ public class MetaDataIndexTemplateServiceTests extends ESTestCase {
assertThat(errors.get(0).getMessage(), equalTo("Alias [foobar] cannot be the same as the template pattern [foobar]"));
}
public void testIndexTemplateWithValidateEmptyMapping() throws Exception {
PutRequest request = new PutRequest("api", "validate_template");
request.template("validate_template");
request.putMapping("type1", "{}");
List<Throwable> errors = putTemplateDetail(request);
assertThat(errors.size(), equalTo(1));
assertThat(errors.get(0), instanceOf(MapperParsingException.class));
assertThat(errors.get(0).getMessage(), containsString("malformed mapping no root object found"));
}
public void testIndexTemplateWithValidateMapping() throws Exception {
PutRequest request = new PutRequest("api", "validate_template");
request.template("te*");
request.putMapping("type1", XContentFactory.jsonBuilder().startObject().startObject("type1").startObject("properties")
.startObject("field2").field("type", "string").field("analyzer", "custom_1").endObject()
.endObject().endObject().endObject().string());
List<Throwable> errors = putTemplateDetail(request);
assertThat(errors.size(), equalTo(1));
assertThat(errors.get(0), instanceOf(MapperParsingException.class));
assertThat(errors.get(0).getMessage(), containsString("analyzer [custom_1] not found for field [field2]"));
}
public void testBrokenMapping() throws Exception {
PutRequest request = new PutRequest("api", "broken_mapping");
request.template("te*");
request.putMapping("type1", "abcde");
List<Throwable> errors = putTemplateDetail(request);
assertThat(errors.size(), equalTo(1));
assertThat(errors.get(0), instanceOf(MapperParsingException.class));
assertThat(errors.get(0).getMessage(), containsString("Failed to parse mapping "));
}
public void testBlankMapping() throws Exception {
PutRequest request = new PutRequest("api", "blank_mapping");
request.template("te*");
request.putMapping("type1", "{}");
List<Throwable> errors = putTemplateDetail(request);
assertThat(errors.size(), equalTo(1));
assertThat(errors.get(0), instanceOf(MapperParsingException.class));
assertThat(errors.get(0).getMessage(), containsString("malformed mapping no root object found"));
}
public void testAliasInvalidFilterInvalidJson() throws Exception {
//invalid json: put index template fails
PutRequest request = new PutRequest("api", "blank_mapping");
request.template("te*");
request.putMapping("type1", "{}");
Set<Alias> aliases = new HashSet<>();
aliases.add(new Alias("invalid_alias").filter("abcde"));
request.aliases(aliases);
List<Throwable> errors = putTemplateDetail(request);
assertThat(errors.size(), equalTo(1));
assertThat(errors.get(0), instanceOf(IllegalArgumentException.class));
assertThat(errors.get(0).getMessage(), equalTo("failed to parse filter for alias [invalid_alias]"));
}
private static List<Throwable> putTemplate(PutRequest request) {
MetaDataCreateIndexService createIndexService = new MetaDataCreateIndexService(
Settings.EMPTY,
@ -94,7 +163,7 @@ public class MetaDataIndexTemplateServiceTests extends ESTestCase {
new HashSet<>(),
null,
null, null);
MetaDataIndexTemplateService service = new MetaDataIndexTemplateService(Settings.EMPTY, null, createIndexService, new AliasValidator(Settings.EMPTY));
MetaDataIndexTemplateService service = new MetaDataIndexTemplateService(Settings.EMPTY, null, createIndexService, new AliasValidator(Settings.EMPTY), null, null);
final List<Throwable> throwables = new ArrayList<>();
service.putTemplate(request, new MetaDataIndexTemplateService.PutListener() {
@ -108,7 +177,42 @@ public class MetaDataIndexTemplateServiceTests extends ESTestCase {
throwables.add(t);
}
});
return throwables;
}
private List<Throwable> putTemplateDetail(PutRequest request) throws Exception {
IndicesService indicesService = getInstanceFromNode(IndicesService.class);
ClusterService clusterService = getInstanceFromNode(ClusterService.class);
NodeServicesProvider nodeServicesProvider = getInstanceFromNode(NodeServicesProvider.class);
MetaDataCreateIndexService createIndexService = new MetaDataCreateIndexService(
Settings.EMPTY,
clusterService,
indicesService,
null,
Version.CURRENT,
null,
new HashSet<>(),
null,
nodeServicesProvider,
null);
MetaDataIndexTemplateService service = new MetaDataIndexTemplateService(
Settings.EMPTY, clusterService, createIndexService, new AliasValidator(Settings.EMPTY), indicesService, nodeServicesProvider);
final List<Throwable> throwables = new ArrayList<>();
final CountDownLatch latch = new CountDownLatch(1);
service.putTemplate(request, new MetaDataIndexTemplateService.PutListener() {
@Override
public void onResponse(MetaDataIndexTemplateService.PutResponse response) {
latch.countDown();
}
@Override
public void onFailure(Throwable t) {
throwables.add(t);
latch.countDown();
}
});
latch.await();
return throwables;
}
}

38
core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java
View File

@ -20,6 +20,7 @@
package org.elasticsearch.cluster.metadata;
import org.elasticsearch.Version;
import org.elasticsearch.cluster.ClusterName;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.EmptyClusterInfoService;
import org.elasticsearch.cluster.block.ClusterBlocks;
@ -31,10 +32,12 @@ import org.elasticsearch.cluster.routing.allocation.AllocationService;
import org.elasticsearch.cluster.routing.allocation.allocator.BalancedShardsAllocator;
import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
import org.elasticsearch.cluster.routing.allocation.decider.MaxRetryAllocationDecider;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.DummyTransportAddress;
import org.elasticsearch.index.IndexNotFoundException;
import org.elasticsearch.indices.IndexAlreadyExistsException;
import org.elasticsearch.indices.InvalidIndexNameException;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.test.gateway.NoopGatewayAllocator;
@ -43,6 +46,7 @@ import java.util.Collections;
import java.util.HashSet;
import static java.util.Collections.emptyMap;
import static org.hamcrest.Matchers.endsWith;
public class MetaDataCreateIndexServiceTests extends ESTestCase {
@ -163,4 +167,38 @@ public class MetaDataCreateIndexServiceTests extends ESTestCase {
return new DiscoveryNode(nodeId, DummyTransportAddress.INSTANCE, emptyMap(),
Collections.unmodifiableSet(new HashSet<>(Arrays.asList(DiscoveryNode.Role.MASTER, DiscoveryNode.Role.DATA))), Version.CURRENT);
}
public void testValidateIndexName() throws Exception {
validateIndexName("index?name", "must not contain the following characters " + Strings.INVALID_FILENAME_CHARS);
validateIndexName("index#name", "must not contain '#'");
validateIndexName("_indexname", "must not start with '_'");
validateIndexName("INDEXNAME", "must be lowercase");
validateIndexName("..", "must not be '.' or '..'");
}
private void validateIndexName(String indexName, String errorMessage) {
InvalidIndexNameException e = expectThrows(InvalidIndexNameException.class,
() -> getCreateIndexService().validateIndexName(indexName, ClusterState.builder(ClusterName.DEFAULT).build()));
assertThat(e.getMessage(), endsWith(errorMessage));
}
private MetaDataCreateIndexService getCreateIndexService() {
return new MetaDataCreateIndexService(
Settings.EMPTY,
null,
null,
null,
Version.CURRENT,
null,
new HashSet<>(),
null,
null,
null);
}
}

23
core/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java
View File

@ -23,7 +23,9 @@ import java.io.IOException;
import java.io.UncheckedIOException;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.concurrent.ExecutionException;
import java.util.function.Function;
@ -38,8 +40,9 @@ import org.elasticsearch.index.mapper.core.NumberFieldMapper.NumberFieldType;
import org.elasticsearch.test.ESSingleNodeTestCase;
import static org.hamcrest.CoreMatchers.containsString;
import static org.hamcrest.Matchers.hasToString;
import static org.hamcrest.Matchers.instanceOf;
import static org.hamcrest.Matchers.startsWith;
public class MapperServiceTests extends ESSingleNodeTestCase {
@ -166,4 +169,22 @@ public class MapperServiceTests extends ESSingleNodeTestCase {
assertThat(mapperService.unmappedFieldType("string"), instanceOf(KeywordFieldType.class));
}
public void testMergeWithMap() throws Throwable {
IndexService indexService1 = createIndex("index1");
MapperService mapperService = indexService1.mapperService();
Map<String, Map<String, Object>> mappings = new HashMap<>();
mappings.put(MapperService.DEFAULT_MAPPING, MapperService.parseMapping("{}"));
MapperException e = expectThrows(MapperParsingException.class,
() -> mapperService.merge(mappings, false));
assertThat(e.getMessage(), startsWith("Failed to parse mapping [" + MapperService.DEFAULT_MAPPING + "]: "));
mappings.clear();
mappings.put("type1", MapperService.parseMapping("{}"));
e = expectThrows( MapperParsingException.class,
() -> mapperService.merge(mappings, false));
assertThat(e.getMessage(), startsWith("Failed to parse mapping [type1]: "));
}
}

123
core/src/test/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java
View File

@ -18,7 +18,6 @@
*/
package org.elasticsearch.indices.template;
import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.action.ActionRequestValidationException;
import org.elasticsearch.action.admin.indices.alias.Alias;
import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse;
@ -32,9 +31,11 @@ import org.elasticsearch.cluster.metadata.AliasMetaData;
import org.elasticsearch.common.ParsingException;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.mapper.MapperParsingException;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.indices.IndexTemplateAlreadyExistsException;
import org.elasticsearch.indices.InvalidAliasNameException;
import org.elasticsearch.indices.InvalidIndexNameException;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.test.ESIntegTestCase;
@ -296,22 +297,15 @@ public class SimpleIndexTemplateIT extends ESIntegTestCase {
GetIndexTemplatesResponse response = client().admin().indices().prepareGetTemplates().get();
assertThat(response.getIndexTemplates(), empty());
client().admin().indices().preparePutTemplate("template_1")
MapperParsingException e = expectThrows( MapperParsingException.class,
() -> client().admin().indices().preparePutTemplate("template_1")
.setTemplate("te*")
.addMapping("type1", "abcde")
.get();
.get());
assertThat(e.getMessage(), containsString("Failed to parse mapping "));
response = client().admin().indices().prepareGetTemplates().get();
assertThat(response.getIndexTemplates(), hasSize(1));
assertThat(response.getIndexTemplates().get(0).getMappings().size(), equalTo(1));
assertThat(response.getIndexTemplates().get(0).getMappings().get("type1").string(), equalTo("abcde"));
try {
createIndex("test");
fail("create index should have failed due to broken index templates mapping");
} catch(ElasticsearchParseException e) {
//everything fine
}
assertThat(response.getIndexTemplates(), hasSize(0));
}
public void testInvalidSettings() throws Exception {
@ -322,15 +316,12 @@ public class SimpleIndexTemplateIT extends ESIntegTestCase {
GetIndexTemplatesResponse response = client().admin().indices().prepareGetTemplates().get();
assertThat(response.getIndexTemplates(), empty());
try {
client().admin().indices().preparePutTemplate("template_1")
IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
() -> client().admin().indices().preparePutTemplate("template_1")
.setTemplate("te*")
.setSettings(Settings.builder().put("does_not_exist", "test"))
.get();
fail();
} catch (IllegalArgumentException ex) {
assertEquals("unknown setting [index.does_not_exist]", ex.getMessage());
}
.get());
assertEquals("unknown setting [index.does_not_exist]", e.getMessage());
response = client().admin().indices().prepareGetTemplates().get();
assertEquals(0, response.getIndexTemplates().size());
@ -490,14 +481,11 @@ public class SimpleIndexTemplateIT extends ESIntegTestCase {
assertThat(response.getIndexTemplates().get(0).getAliases().size(), equalTo(1));
assertThat(response.getIndexTemplates().get(0).getAliases().get("invalid_alias").filter().string(), equalTo("{\"invalid\":{}}"));
try {
createIndex("test");
fail("index creation should have failed due to invalid alias filter in matching index template");
} catch(IllegalArgumentException e) {
assertThat(e.getMessage(), equalTo("failed to parse filter for alias [invalid_alias]"));
assertThat(e.getCause(), instanceOf(ParsingException.class));
assertThat(e.getCause().getMessage(), equalTo("no [query] registered for [invalid]"));
}
IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
() -> createIndex("test"));
assertThat(e.getMessage(), equalTo("failed to parse filter for alias [invalid_alias]"));
assertThat(e.getCause(), instanceOf(ParsingException.class));
assertThat(e.getCause().getMessage(), equalTo("no [query] registered for [invalid]"));
}
public void testAliasInvalidFilterInvalidJson() throws Exception {
@ -506,11 +494,9 @@ public class SimpleIndexTemplateIT extends ESIntegTestCase {
.setTemplate("te*")
.addAlias(new Alias("invalid_alias").filter("abcde"));
try {
putIndexTemplateRequestBuilder.get();
} catch(IllegalArgumentException e) {
assertThat(e.getMessage(), equalTo("failed to parse filter for alias [invalid_alias]"));
}
IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
() -> putIndexTemplateRequestBuilder.get());
assertThat(e.getMessage(), equalTo("failed to parse filter for alias [invalid_alias]"));
GetIndexTemplatesResponse response = client().admin().indices().prepareGetTemplates("template_1").get();
assertThat(response.getIndexTemplates().size(), equalTo(0));
@ -523,12 +509,9 @@ public class SimpleIndexTemplateIT extends ESIntegTestCase {
.setTemplate("te*")
.addAlias(new Alias("index")).get();
try {
createIndex("test");
fail("index creation should have failed due to alias with existing index name in matching index template");
} catch(InvalidAliasNameException e) {
assertThat(e.getMessage(), equalTo("Invalid alias name [index], an index exists with the same name as the alias"));
}
InvalidAliasNameException e = expectThrows(InvalidAliasNameException.class,
() -> createIndex("test"));
assertThat(e.getMessage(), equalTo("Invalid alias name [index], an index exists with the same name as the alias"));
}
public void testAliasEmptyName() throws Exception {
@ -536,12 +519,9 @@ public class SimpleIndexTemplateIT extends ESIntegTestCase {
.setTemplate("te*")
.addAlias(new Alias(" ").indexRouting("1,2,3"));
try {
putIndexTemplateRequestBuilder.get();
fail("put template should have failed due to alias with empty name");
} catch (IllegalArgumentException e) {
assertThat(e.getMessage(), equalTo("alias name is required"));
}
IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
() -> putIndexTemplateRequestBuilder.get());
assertThat(e.getMessage(), equalTo("alias name is required"));
}
public void testAliasWithMultipleIndexRoutings() throws Exception {
@ -549,12 +529,9 @@ public class SimpleIndexTemplateIT extends ESIntegTestCase {
.setTemplate("te*")
.addAlias(new Alias("alias").indexRouting("1,2,3"));
try {
putIndexTemplateRequestBuilder.get();
fail("put template should have failed due to alias with multiple index routings");
} catch (IllegalArgumentException e) {
assertThat(e.getMessage(), equalTo("alias [alias] has several index routing values associated with it"));
}
IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
() -> putIndexTemplateRequestBuilder.get());
assertThat(e.getMessage(), equalTo("alias [alias] has several index routing values associated with it"));
}
public void testMultipleAliasesPrecedence() throws Exception {
@ -664,4 +641,48 @@ public class SimpleIndexTemplateIT extends ESIntegTestCase {
assertThat(response.getItems()[0].getVersion(), equalTo(1L));
}
public void testCombineTemplates() throws Exception{
// clean all templates setup by the framework.
client().admin().indices().prepareDeleteTemplate("*").get();
// check get all templates on an empty index.
GetIndexTemplatesResponse response = client().admin().indices().prepareGetTemplates().get();
assertThat(response.getIndexTemplates(), empty());
//Now, a complete mapping with two separated templates is error
// base template
client().admin().indices().preparePutTemplate("template_1")
.setTemplate("*")
.setSettings(
" {\n" +
" \"index\" : {\n" +
" \"analysis\" : {\n" +
" \"analyzer\" : {\n" +
" \"custom_1\" : {\n" +
" \"tokenizer\" : \"whitespace\"\n" +
" }\n" +
" }\n" +
" }\n" +
" }\n" +
" }\n")
.get();
// put template using custom_1 analyzer
MapperParsingException e = expectThrows(MapperParsingException.class,
() -> client().admin().indices().preparePutTemplate("template_2")
.setTemplate("test*")
.setCreate(true)
.setOrder(1)
.addMapping("type1", XContentFactory.jsonBuilder().startObject().startObject("type1").startObject("properties")
.startObject("field2").field("type", "string").field("analyzer", "custom_1").endObject()
.endObject().endObject().endObject())
.get());
assertThat(e.getMessage(), containsString("analyzer [custom_1] not found for field [field2]"));
response = client().admin().indices().prepareGetTemplates().get();
assertThat(response.getIndexTemplates(), hasSize(1));
}
}

11
core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java
View File

@ -59,6 +59,7 @@ import org.elasticsearch.common.unit.ByteSizeUnit;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.index.store.IndexStore;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.indices.InvalidIndexNameException;
import org.elasticsearch.repositories.RepositoriesService;
import org.elasticsearch.repositories.RepositoryException;
@ -416,7 +417,10 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
.setType("fs").setSettings(Settings.builder().put("location", randomRepoPath())));
logger.info("--> creating test template");
assertThat(client.admin().indices().preparePutTemplate("test-template").setTemplate("te*").addMapping("test-mapping", "{}").get().isAcknowledged(), equalTo(true));
assertThat(client.admin().indices().preparePutTemplate("test-template").setTemplate("te*").addMapping("test-mapping", XContentFactory.jsonBuilder().startObject().startObject("test-mapping").startObject("properties")
.startObject("field1").field("type", "string").field("store", "yes").endObject()
.startObject("field2").field("type", "string").field("store", "yes").field("index", "not_analyzed").endObject()
.endObject().endObject().endObject()).get().isAcknowledged(), equalTo(true));
logger.info("--> snapshot");
CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap").setIndices().setWaitForCompletion(true).get();
@ -449,7 +453,10 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
.setType("fs").setSettings(Settings.builder().put("location", location)));
logger.info("--> creating test template");
assertThat(client.admin().indices().preparePutTemplate("test-template").setTemplate("te*").addMapping("test-mapping", "{}").get().isAcknowledged(), equalTo(true));
assertThat(client.admin().indices().preparePutTemplate("test-template").setTemplate("te*").addMapping("test-mapping", XContentFactory.jsonBuilder().startObject().startObject("test-mapping").startObject("properties")
.startObject("field1").field("type", "string").field("store", "yes").endObject()
.startObject("field2").field("type", "string").field("store", "yes").field("index", "not_analyzed").endObject()
.endObject().endObject().endObject()).get().isAcknowledged(), equalTo(true));
logger.info("--> snapshot without global state");
CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap-no-global-state").setIndices().setIncludeGlobalState(false).setWaitForCompletion(true).get();