Index Template: parse and validate mappings in template when template puts
Share applying template with MetaDataCreateIndexService and MetaDataIndexTemplateService Add some unit test Collapse addMappingsToMapperService and move it to MapperService Extract validateTemplate method use expectThrows in testcase Add TODO comment Closes #2415
This commit is contained in:
Jun Ohtani
parent
93c5a9dacd
commit
fd76291131
|
|
@ -55,9 +55,7 @@ import org.elasticsearch.common.io.PathUtils;
|
|||
import org.elasticsearch.common.regex.Regex;
|
||||
import org.elasticsearch.common.settings.IndexScopedSettings;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.common.xcontent.XContentFactory;
|
||||
import org.elasticsearch.common.xcontent.XContentHelper;
|
||||
import org.elasticsearch.common.xcontent.XContentParser;
|
||||
import org.elasticsearch.env.Environment;
|
||||
import org.elasticsearch.index.Index;
|
||||
import org.elasticsearch.index.IndexNotFoundException;
|
||||
|
|
@ -220,7 +218,7 @@ public class MetaDataCreateIndexService extends AbstractComponent {
|
|||
List<String> templateNames = new ArrayList<>();
|
||||
|
||||
for (Map.Entry<String, String> entry : request.mappings().entrySet()) {
|
||||
mappings.put(entry.getKey(), parseMapping(entry.getValue()));
|
||||
mappings.put(entry.getKey(), MapperService.parseMapping(entry.getValue()));
|
||||
}
|
||||
|
||||
for (Map.Entry<String, Custom> entry : request.customs().entrySet()) {
|
||||
|
|
@ -232,9 +230,9 @@ public class MetaDataCreateIndexService extends AbstractComponent {
|
|||
templateNames.add(template.getName());
|
||||
for (ObjectObjectCursor<String, CompressedXContent> cursor : template.mappings()) {
|
||||
if (mappings.containsKey(cursor.key)) {
|
||||
XContentHelper.mergeDefaults(mappings.get(cursor.key), parseMapping(cursor.value.string()));
|
||||
XContentHelper.mergeDefaults(mappings.get(cursor.key), MapperService.parseMapping(cursor.value.string()));
|
||||
} else {
|
||||
mappings.put(cursor.key, parseMapping(cursor.value.string()));
|
||||
mappings.put(cursor.key, MapperService.parseMapping(cursor.value.string()));
|
||||
}
|
||||
}
|
||||
// handle custom
|
||||
|
|
@ -315,26 +313,11 @@ public class MetaDataCreateIndexService extends AbstractComponent {
|
|||
createdIndex = indexService.index();
|
||||
// now add the mappings
|
||||
MapperService mapperService = indexService.mapperService();
|
||||
// first, add the default mapping
|
||||
if (mappings.containsKey(MapperService.DEFAULT_MAPPING)) {
|
||||
try {
|
||||
mapperService.merge(MapperService.DEFAULT_MAPPING, new CompressedXContent(XContentFactory.jsonBuilder().map(mappings.get(MapperService.DEFAULT_MAPPING)).string()), MapperService.MergeReason.MAPPING_UPDATE, request.updateAllTypes());
|
||||
} catch (Exception e) {
|
||||
removalReason = "failed on parsing default mapping on index creation";
|
||||
throw new MapperParsingException("Failed to parse mapping [{}]: {}", e, MapperService.DEFAULT_MAPPING, e.getMessage());
|
||||
}
|
||||
}
|
||||
for (Map.Entry<String, Map<String, Object>> entry : mappings.entrySet()) {
|
||||
if (entry.getKey().equals(MapperService.DEFAULT_MAPPING)) {
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
// apply the default here, its the first time we parse it
|
||||
mapperService.merge(entry.getKey(), new CompressedXContent(XContentFactory.jsonBuilder().map(entry.getValue()).string()), MapperService.MergeReason.MAPPING_UPDATE, request.updateAllTypes());
|
||||
} catch (Exception e) {
|
||||
removalReason = "failed on parsing mappings on index creation";
|
||||
throw new MapperParsingException("Failed to parse mapping [{}]: {}", e, entry.getKey(), e.getMessage());
|
||||
}
|
||||
mapperService.merge(mappings, request.updateAllTypes());
|
||||
} catch (MapperParsingException mpe) {
|
||||
removalReason = "failed on parsing default mapping/mappings on index creation";
|
||||
throw mpe;
|
||||
}
|
||||
|
||||
final QueryShardContext queryShardContext = indexService.newQueryShardContext();
|
||||
|
|
@ -426,12 +409,6 @@ public class MetaDataCreateIndexService extends AbstractComponent {
|
|||
});
|
||||
}
|
||||
|
||||
private Map<String, Object> parseMapping(String mappingSource) throws Exception {
|
||||
try (XContentParser parser = XContentFactory.xContent(mappingSource).createParser(mappingSource)) {
|
||||
return parser.map();
|
||||
}
|
||||
}
|
||||
|
||||
private List<IndexTemplateMetaData> findTemplates(CreateIndexClusterStateUpdateRequest request, ClusterState state, IndexTemplateFilter indexTemplateFilter) throws IOException {
|
||||
List<IndexTemplateMetaData> templates = new ArrayList<>();
|
||||
for (ObjectCursor<IndexTemplateMetaData> cursor : state.metaData().templates().values()) {
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@
|
|||
package org.elasticsearch.cluster.metadata;
|
||||
|
||||
import com.carrotsearch.hppc.cursors.ObjectCursor;
|
||||
import org.elasticsearch.Version;
|
||||
import org.elasticsearch.action.admin.indices.alias.Alias;
|
||||
import org.elasticsearch.action.support.master.MasterNodeRequest;
|
||||
import org.elasticsearch.cluster.ClusterState;
|
||||
|
|
@ -26,17 +27,25 @@ import org.elasticsearch.cluster.ClusterStateUpdateTask;
|
|||
import org.elasticsearch.cluster.service.ClusterService;
|
||||
import org.elasticsearch.common.Priority;
|
||||
import org.elasticsearch.common.Strings;
|
||||
import org.elasticsearch.common.UUIDs;
|
||||
import org.elasticsearch.common.ValidationException;
|
||||
import org.elasticsearch.common.component.AbstractComponent;
|
||||
import org.elasticsearch.common.inject.Inject;
|
||||
import org.elasticsearch.common.regex.Regex;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.common.unit.TimeValue;
|
||||
import org.elasticsearch.index.Index;
|
||||
import org.elasticsearch.index.NodeServicesProvider;
|
||||
import org.elasticsearch.index.IndexService;
|
||||
import org.elasticsearch.index.mapper.MapperParsingException;
|
||||
import org.elasticsearch.index.mapper.MapperService;
|
||||
import org.elasticsearch.indices.IndexTemplateAlreadyExistsException;
|
||||
import org.elasticsearch.indices.IndexTemplateMissingException;
|
||||
import org.elasticsearch.indices.IndicesService;
|
||||
import org.elasticsearch.indices.InvalidIndexTemplateException;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
|
|
@ -51,14 +60,18 @@ public class MetaDataIndexTemplateService extends AbstractComponent {
|
|||
|
||||
private final ClusterService clusterService;
|
||||
private final AliasValidator aliasValidator;
|
||||
private final IndicesService indicesService;
|
||||
private final MetaDataCreateIndexService metaDataCreateIndexService;
|
||||
private final NodeServicesProvider nodeServicesProvider;
|
||||
|
||||
@Inject
|
||||
public MetaDataIndexTemplateService(Settings settings, ClusterService clusterService, MetaDataCreateIndexService metaDataCreateIndexService, AliasValidator aliasValidator) {
|
||||
public MetaDataIndexTemplateService(Settings settings, ClusterService clusterService, MetaDataCreateIndexService metaDataCreateIndexService, AliasValidator aliasValidator, IndicesService indicesService, NodeServicesProvider nodeServicesProvider) {
|
||||
super(settings);
|
||||
this.clusterService = clusterService;
|
||||
this.aliasValidator = aliasValidator;
|
||||
this.indicesService = indicesService;
|
||||
this.metaDataCreateIndexService = metaDataCreateIndexService;
|
||||
this.nodeServicesProvider = nodeServicesProvider;
|
||||
}
|
||||
|
||||
public void removeTemplates(final RemoveRequest request, final RemoveListener listener) {
|
||||
|
|
@ -126,28 +139,7 @@ public class MetaDataIndexTemplateService extends AbstractComponent {
|
|||
return;
|
||||
}
|
||||
|
||||
IndexTemplateMetaData.Builder templateBuilder;
|
||||
try {
|
||||
templateBuilder = IndexTemplateMetaData.builder(request.name);
|
||||
templateBuilder.order(request.order);
|
||||
templateBuilder.template(request.template);
|
||||
templateBuilder.settings(request.settings);
|
||||
for (Map.Entry<String, String> entry : request.mappings.entrySet()) {
|
||||
templateBuilder.putMapping(entry.getKey(), entry.getValue());
|
||||
}
|
||||
for (Alias alias : request.aliases) {
|
||||
AliasMetaData aliasMetaData = AliasMetaData.builder(alias.name()).filter(alias.filter())
|
||||
.indexRouting(alias.indexRouting()).searchRouting(alias.searchRouting()).build();
|
||||
templateBuilder.putAlias(aliasMetaData);
|
||||
}
|
||||
for (Map.Entry<String, IndexMetaData.Custom> entry : request.customs.entrySet()) {
|
||||
templateBuilder.putCustom(entry.getKey(), entry.getValue());
|
||||
}
|
||||
} catch (Throwable e) {
|
||||
listener.onFailure(e);
|
||||
return;
|
||||
}
|
||||
final IndexTemplateMetaData template = templateBuilder.build();
|
||||
final IndexTemplateMetaData.Builder templateBuilder = IndexTemplateMetaData.builder(request.name);
|
||||
|
||||
clusterService.submitStateUpdateTask("create-index-template [" + request.name + "], cause [" + request.cause + "]",
|
||||
new ClusterStateUpdateTask(Priority.URGENT) {
|
||||
|
|
@ -163,10 +155,23 @@ public class MetaDataIndexTemplateService extends AbstractComponent {
|
|||
}
|
||||
|
||||
@Override
|
||||
public ClusterState execute(ClusterState currentState) {
|
||||
public ClusterState execute(ClusterState currentState) throws Exception {
|
||||
if (request.create && currentState.metaData().templates().containsKey(request.name)) {
|
||||
throw new IndexTemplateAlreadyExistsException(request.name);
|
||||
}
|
||||
|
||||
validateAndAddTemplate(request, templateBuilder, indicesService, nodeServicesProvider, metaDataCreateIndexService);
|
||||
|
||||
for (Alias alias : request.aliases) {
|
||||
AliasMetaData aliasMetaData = AliasMetaData.builder(alias.name()).filter(alias.filter())
|
||||
.indexRouting(alias.indexRouting()).searchRouting(alias.searchRouting()).build();
|
||||
templateBuilder.putAlias(aliasMetaData);
|
||||
}
|
||||
for (Map.Entry<String, IndexMetaData.Custom> entry : request.customs.entrySet()) {
|
||||
templateBuilder.putCustom(entry.getKey(), entry.getValue());
|
||||
}
|
||||
IndexTemplateMetaData template = templateBuilder.build();
|
||||
|
||||
MetaData.Builder builder = MetaData.builder(currentState.metaData()).put(template);
|
||||
|
||||
return ClusterState.builder(currentState).metaData(builder).build();
|
||||
|
|
@ -174,11 +179,53 @@ public class MetaDataIndexTemplateService extends AbstractComponent {
|
|||
|
||||
@Override
|
||||
public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
|
||||
listener.onResponse(new PutResponse(true, template));
|
||||
listener.onResponse(new PutResponse(true, templateBuilder.build()));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private static void validateAndAddTemplate(final PutRequest request, IndexTemplateMetaData.Builder templateBuilder, IndicesService indicesService,
|
||||
NodeServicesProvider nodeServicesProvider, MetaDataCreateIndexService metaDataCreateIndexService) throws Exception {
|
||||
Index createdIndex = null;
|
||||
final String temporaryIndexName = UUIDs.randomBase64UUID();
|
||||
try {
|
||||
|
||||
//create index service for parsing and validating "mappings"
|
||||
Settings dummySettings = Settings.builder()
|
||||
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
|
||||
.put(request.settings)
|
||||
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
|
||||
.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
|
||||
.put(IndexMetaData.SETTING_INDEX_UUID, UUIDs.randomBase64UUID())
|
||||
.build();
|
||||
|
||||
final IndexMetaData tmpIndexMetadata = IndexMetaData.builder(temporaryIndexName).settings(dummySettings).build();
|
||||
IndexService dummyIndexService = indicesService.createIndex(nodeServicesProvider, tmpIndexMetadata, Collections.emptyList());
|
||||
createdIndex = dummyIndexService.index();
|
||||
|
||||
templateBuilder.order(request.order);
|
||||
templateBuilder.template(request.template);
|
||||
templateBuilder.settings(request.settings);
|
||||
|
||||
Map<String, Map<String, Object>> mappingsForValidation = new HashMap<>();
|
||||
for (Map.Entry<String, String> entry : request.mappings.entrySet()) {
|
||||
try {
|
||||
templateBuilder.putMapping(entry.getKey(), entry.getValue());
|
||||
} catch (Exception e) {
|
||||
throw new MapperParsingException("Failed to parse mapping [{}]: {}", e, entry.getKey(), e.getMessage());
|
||||
}
|
||||
mappingsForValidation.put(entry.getKey(), MapperService.parseMapping(entry.getValue()));
|
||||
}
|
||||
|
||||
dummyIndexService.mapperService().merge(mappingsForValidation, false);
|
||||
|
||||
} finally {
|
||||
if (createdIndex != null) {
|
||||
indicesService.removeIndex(createdIndex, " created for parsing template mapping");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void validate(PutRequest request) {
|
||||
List<String> validationErrors = new ArrayList<>();
|
||||
if (request.name.contains(" ")) {
|
||||
|
|
|
|||
|
|
@ -29,6 +29,8 @@ import org.elasticsearch.common.compress.CompressedXContent;
|
|||
import org.elasticsearch.common.regex.Regex;
|
||||
import org.elasticsearch.common.settings.Setting;
|
||||
import org.elasticsearch.common.settings.Setting.Property;
|
||||
import org.elasticsearch.common.xcontent.XContentFactory;
|
||||
import org.elasticsearch.common.xcontent.XContentParser;
|
||||
import org.elasticsearch.index.AbstractIndexComponent;
|
||||
import org.elasticsearch.index.IndexSettings;
|
||||
import org.elasticsearch.index.analysis.AnalysisService;
|
||||
|
|
@ -175,6 +177,35 @@ public class MapperService extends AbstractIndexComponent {
|
|||
return this.documentParser;
|
||||
}
|
||||
|
||||
public static Map<String, Object> parseMapping(String mappingSource) throws Exception {
|
||||
try (XContentParser parser = XContentFactory.xContent(mappingSource).createParser(mappingSource)) {
|
||||
return parser.map();
|
||||
}
|
||||
}
|
||||
|
||||
//TODO: make this atomic
|
||||
public void merge(Map<String, Map<String, Object>> mappings, boolean updateAllTypes) throws MapperParsingException {
|
||||
// first, add the default mapping
|
||||
if (mappings.containsKey(DEFAULT_MAPPING)) {
|
||||
try {
|
||||
this.merge(DEFAULT_MAPPING, new CompressedXContent(XContentFactory.jsonBuilder().map(mappings.get(DEFAULT_MAPPING)).string()), MergeReason.MAPPING_UPDATE, updateAllTypes);
|
||||
} catch (Exception e) {
|
||||
throw new MapperParsingException("Failed to parse mapping [{}]: {}", e, DEFAULT_MAPPING, e.getMessage());
|
||||
}
|
||||
}
|
||||
for (Map.Entry<String, Map<String, Object>> entry : mappings.entrySet()) {
|
||||
if (entry.getKey().equals(DEFAULT_MAPPING)) {
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
// apply the default here, its the first time we parse it
|
||||
this.merge(entry.getKey(), new CompressedXContent(XContentFactory.jsonBuilder().map(entry.getValue()).string()), MergeReason.MAPPING_UPDATE, updateAllTypes);
|
||||
} catch (Exception e) {
|
||||
throw new MapperParsingException("Failed to parse mapping [{}]: {}", e, entry.getKey(), e.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public DocumentMapper merge(String type, CompressedXContent mappingSource, MergeReason reason, boolean updateAllTypes) {
|
||||
if (DEFAULT_MAPPING.equals(type)) {
|
||||
// verify we can parse it
|
||||
|
|
|
|||
|
|
@ -26,9 +26,14 @@ import org.elasticsearch.cluster.metadata.IndexMetaData;
|
|||
import org.elasticsearch.cluster.metadata.MetaDataCreateIndexService;
|
||||
import org.elasticsearch.cluster.metadata.MetaDataIndexTemplateService;
|
||||
import org.elasticsearch.cluster.metadata.MetaDataIndexTemplateService.PutRequest;
|
||||
import org.elasticsearch.cluster.service.ClusterService;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.common.xcontent.XContentFactory;
|
||||
import org.elasticsearch.index.NodeServicesProvider;
|
||||
import org.elasticsearch.index.mapper.MapperParsingException;
|
||||
import org.elasticsearch.indices.IndicesService;
|
||||
import org.elasticsearch.indices.InvalidIndexTemplateException;
|
||||
import org.elasticsearch.test.ESTestCase;
|
||||
import org.elasticsearch.test.ESSingleNodeTestCase;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
|
|
@ -36,12 +41,14 @@ import java.util.HashMap;
|
|||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
|
||||
import static org.hamcrest.CoreMatchers.containsString;
|
||||
import static org.hamcrest.CoreMatchers.equalTo;
|
||||
import static org.hamcrest.CoreMatchers.instanceOf;
|
||||
|
||||
public class MetaDataIndexTemplateServiceTests extends ESTestCase {
|
||||
public class MetaDataIndexTemplateServiceTests extends ESSingleNodeTestCase {
|
||||
public void testIndexTemplateInvalidNumberOfShards() {
|
||||
PutRequest request = new PutRequest("test", "test_shards");
|
||||
request.template("test_shards*");
|
||||
|
|
@ -83,6 +90,68 @@ public class MetaDataIndexTemplateServiceTests extends ESTestCase {
|
|||
assertThat(errors.get(0).getMessage(), equalTo("Alias [foobar] cannot be the same as the template pattern [foobar]"));
|
||||
}
|
||||
|
||||
public void testIndexTemplateWithValidateEmptyMapping() throws Exception {
|
||||
PutRequest request = new PutRequest("api", "validate_template");
|
||||
request.template("validate_template");
|
||||
request.putMapping("type1", "{}");
|
||||
|
||||
List<Throwable> errors = putTemplateDetail(request);
|
||||
assertThat(errors.size(), equalTo(1));
|
||||
assertThat(errors.get(0), instanceOf(MapperParsingException.class));
|
||||
assertThat(errors.get(0).getMessage(), containsString("malformed mapping no root object found"));
|
||||
}
|
||||
|
||||
public void testIndexTemplateWithValidateMapping() throws Exception {
|
||||
PutRequest request = new PutRequest("api", "validate_template");
|
||||
request.template("te*");
|
||||
request.putMapping("type1", XContentFactory.jsonBuilder().startObject().startObject("type1").startObject("properties")
|
||||
.startObject("field2").field("type", "string").field("analyzer", "custom_1").endObject()
|
||||
.endObject().endObject().endObject().string());
|
||||
|
||||
List<Throwable> errors = putTemplateDetail(request);
|
||||
assertThat(errors.size(), equalTo(1));
|
||||
assertThat(errors.get(0), instanceOf(MapperParsingException.class));
|
||||
assertThat(errors.get(0).getMessage(), containsString("analyzer [custom_1] not found for field [field2]"));
|
||||
}
|
||||
|
||||
public void testBrokenMapping() throws Exception {
|
||||
PutRequest request = new PutRequest("api", "broken_mapping");
|
||||
request.template("te*");
|
||||
request.putMapping("type1", "abcde");
|
||||
|
||||
List<Throwable> errors = putTemplateDetail(request);
|
||||
assertThat(errors.size(), equalTo(1));
|
||||
assertThat(errors.get(0), instanceOf(MapperParsingException.class));
|
||||
assertThat(errors.get(0).getMessage(), containsString("Failed to parse mapping "));
|
||||
}
|
||||
|
||||
public void testBlankMapping() throws Exception {
|
||||
PutRequest request = new PutRequest("api", "blank_mapping");
|
||||
request.template("te*");
|
||||
request.putMapping("type1", "{}");
|
||||
|
||||
List<Throwable> errors = putTemplateDetail(request);
|
||||
assertThat(errors.size(), equalTo(1));
|
||||
assertThat(errors.get(0), instanceOf(MapperParsingException.class));
|
||||
assertThat(errors.get(0).getMessage(), containsString("malformed mapping no root object found"));
|
||||
}
|
||||
|
||||
public void testAliasInvalidFilterInvalidJson() throws Exception {
|
||||
//invalid json: put index template fails
|
||||
PutRequest request = new PutRequest("api", "blank_mapping");
|
||||
request.template("te*");
|
||||
request.putMapping("type1", "{}");
|
||||
Set<Alias> aliases = new HashSet<>();
|
||||
aliases.add(new Alias("invalid_alias").filter("abcde"));
|
||||
request.aliases(aliases);
|
||||
|
||||
List<Throwable> errors = putTemplateDetail(request);
|
||||
assertThat(errors.size(), equalTo(1));
|
||||
assertThat(errors.get(0), instanceOf(IllegalArgumentException.class));
|
||||
assertThat(errors.get(0).getMessage(), equalTo("failed to parse filter for alias [invalid_alias]"));
|
||||
}
|
||||
|
||||
|
||||
private static List<Throwable> putTemplate(PutRequest request) {
|
||||
MetaDataCreateIndexService createIndexService = new MetaDataCreateIndexService(
|
||||
Settings.EMPTY,
|
||||
|
|
@ -94,7 +163,7 @@ public class MetaDataIndexTemplateServiceTests extends ESTestCase {
|
|||
new HashSet<>(),
|
||||
null,
|
||||
null, null);
|
||||
MetaDataIndexTemplateService service = new MetaDataIndexTemplateService(Settings.EMPTY, null, createIndexService, new AliasValidator(Settings.EMPTY));
|
||||
MetaDataIndexTemplateService service = new MetaDataIndexTemplateService(Settings.EMPTY, null, createIndexService, new AliasValidator(Settings.EMPTY), null, null);
|
||||
|
||||
final List<Throwable> throwables = new ArrayList<>();
|
||||
service.putTemplate(request, new MetaDataIndexTemplateService.PutListener() {
|
||||
|
|
@ -108,7 +177,42 @@ public class MetaDataIndexTemplateServiceTests extends ESTestCase {
|
|||
throwables.add(t);
|
||||
}
|
||||
});
|
||||
return throwables;
|
||||
}
|
||||
|
||||
private List<Throwable> putTemplateDetail(PutRequest request) throws Exception {
|
||||
IndicesService indicesService = getInstanceFromNode(IndicesService.class);
|
||||
ClusterService clusterService = getInstanceFromNode(ClusterService.class);
|
||||
NodeServicesProvider nodeServicesProvider = getInstanceFromNode(NodeServicesProvider.class);
|
||||
MetaDataCreateIndexService createIndexService = new MetaDataCreateIndexService(
|
||||
Settings.EMPTY,
|
||||
clusterService,
|
||||
indicesService,
|
||||
null,
|
||||
Version.CURRENT,
|
||||
null,
|
||||
new HashSet<>(),
|
||||
null,
|
||||
nodeServicesProvider,
|
||||
null);
|
||||
MetaDataIndexTemplateService service = new MetaDataIndexTemplateService(
|
||||
Settings.EMPTY, clusterService, createIndexService, new AliasValidator(Settings.EMPTY), indicesService, nodeServicesProvider);
|
||||
|
||||
final List<Throwable> throwables = new ArrayList<>();
|
||||
final CountDownLatch latch = new CountDownLatch(1);
|
||||
service.putTemplate(request, new MetaDataIndexTemplateService.PutListener() {
|
||||
@Override
|
||||
public void onResponse(MetaDataIndexTemplateService.PutResponse response) {
|
||||
latch.countDown();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onFailure(Throwable t) {
|
||||
throwables.add(t);
|
||||
latch.countDown();
|
||||
}
|
||||
});
|
||||
latch.await();
|
||||
return throwables;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@
|
|||
package org.elasticsearch.cluster.metadata;
|
||||
|
||||
import org.elasticsearch.Version;
|
||||
import org.elasticsearch.cluster.ClusterName;
|
||||
import org.elasticsearch.cluster.ClusterState;
|
||||
import org.elasticsearch.cluster.EmptyClusterInfoService;
|
||||
import org.elasticsearch.cluster.block.ClusterBlocks;
|
||||
|
|
@ -31,10 +32,12 @@ import org.elasticsearch.cluster.routing.allocation.AllocationService;
|
|||
import org.elasticsearch.cluster.routing.allocation.allocator.BalancedShardsAllocator;
|
||||
import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
|
||||
import org.elasticsearch.cluster.routing.allocation.decider.MaxRetryAllocationDecider;
|
||||
import org.elasticsearch.common.Strings;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.common.transport.DummyTransportAddress;
|
||||
import org.elasticsearch.index.IndexNotFoundException;
|
||||
import org.elasticsearch.indices.IndexAlreadyExistsException;
|
||||
import org.elasticsearch.indices.InvalidIndexNameException;
|
||||
import org.elasticsearch.test.ESTestCase;
|
||||
import org.elasticsearch.test.gateway.NoopGatewayAllocator;
|
||||
|
||||
|
|
@ -43,6 +46,7 @@ import java.util.Collections;
|
|||
import java.util.HashSet;
|
||||
|
||||
import static java.util.Collections.emptyMap;
|
||||
import static org.hamcrest.Matchers.endsWith;
|
||||
|
||||
public class MetaDataCreateIndexServiceTests extends ESTestCase {
|
||||
|
||||
|
|
@ -163,4 +167,38 @@ public class MetaDataCreateIndexServiceTests extends ESTestCase {
|
|||
return new DiscoveryNode(nodeId, DummyTransportAddress.INSTANCE, emptyMap(),
|
||||
Collections.unmodifiableSet(new HashSet<>(Arrays.asList(DiscoveryNode.Role.MASTER, DiscoveryNode.Role.DATA))), Version.CURRENT);
|
||||
}
|
||||
|
||||
public void testValidateIndexName() throws Exception {
|
||||
|
||||
validateIndexName("index?name", "must not contain the following characters " + Strings.INVALID_FILENAME_CHARS);
|
||||
|
||||
validateIndexName("index#name", "must not contain '#'");
|
||||
|
||||
validateIndexName("_indexname", "must not start with '_'");
|
||||
|
||||
validateIndexName("INDEXNAME", "must be lowercase");
|
||||
|
||||
validateIndexName("..", "must not be '.' or '..'");
|
||||
|
||||
}
|
||||
|
||||
private void validateIndexName(String indexName, String errorMessage) {
|
||||
InvalidIndexNameException e = expectThrows(InvalidIndexNameException.class,
|
||||
() -> getCreateIndexService().validateIndexName(indexName, ClusterState.builder(ClusterName.DEFAULT).build()));
|
||||
assertThat(e.getMessage(), endsWith(errorMessage));
|
||||
}
|
||||
|
||||
private MetaDataCreateIndexService getCreateIndexService() {
|
||||
return new MetaDataCreateIndexService(
|
||||
Settings.EMPTY,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
Version.CURRENT,
|
||||
null,
|
||||
new HashSet<>(),
|
||||
null,
|
||||
null,
|
||||
null);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -23,7 +23,9 @@ import java.io.IOException;
|
|||
import java.io.UncheckedIOException;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.function.Function;
|
||||
|
||||
|
|
@ -38,8 +40,9 @@ import org.elasticsearch.index.mapper.core.NumberFieldMapper.NumberFieldType;
|
|||
import org.elasticsearch.test.ESSingleNodeTestCase;
|
||||
|
||||
import static org.hamcrest.CoreMatchers.containsString;
|
||||
import static org.hamcrest.Matchers.hasToString;
|
||||
import static org.hamcrest.Matchers.instanceOf;
|
||||
|
||||
import static org.hamcrest.Matchers.startsWith;
|
||||
|
||||
public class MapperServiceTests extends ESSingleNodeTestCase {
|
||||
|
||||
|
|
@ -166,4 +169,22 @@ public class MapperServiceTests extends ESSingleNodeTestCase {
|
|||
assertThat(mapperService.unmappedFieldType("string"), instanceOf(KeywordFieldType.class));
|
||||
}
|
||||
|
||||
|
||||
public void testMergeWithMap() throws Throwable {
|
||||
IndexService indexService1 = createIndex("index1");
|
||||
MapperService mapperService = indexService1.mapperService();
|
||||
Map<String, Map<String, Object>> mappings = new HashMap<>();
|
||||
|
||||
mappings.put(MapperService.DEFAULT_MAPPING, MapperService.parseMapping("{}"));
|
||||
MapperException e = expectThrows(MapperParsingException.class,
|
||||
() -> mapperService.merge(mappings, false));
|
||||
assertThat(e.getMessage(), startsWith("Failed to parse mapping [" + MapperService.DEFAULT_MAPPING + "]: "));
|
||||
|
||||
mappings.clear();
|
||||
mappings.put("type1", MapperService.parseMapping("{}"));
|
||||
|
||||
e = expectThrows( MapperParsingException.class,
|
||||
() -> mapperService.merge(mappings, false));
|
||||
assertThat(e.getMessage(), startsWith("Failed to parse mapping [type1]: "));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -18,7 +18,6 @@
|
|||
*/
|
||||
package org.elasticsearch.indices.template;
|
||||
|
||||
import org.elasticsearch.ElasticsearchParseException;
|
||||
import org.elasticsearch.action.ActionRequestValidationException;
|
||||
import org.elasticsearch.action.admin.indices.alias.Alias;
|
||||
import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse;
|
||||
|
|
@ -32,9 +31,11 @@ import org.elasticsearch.cluster.metadata.AliasMetaData;
|
|||
import org.elasticsearch.common.ParsingException;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.common.xcontent.XContentFactory;
|
||||
import org.elasticsearch.index.mapper.MapperParsingException;
|
||||
import org.elasticsearch.index.query.QueryBuilders;
|
||||
import org.elasticsearch.indices.IndexTemplateAlreadyExistsException;
|
||||
import org.elasticsearch.indices.InvalidAliasNameException;
|
||||
import org.elasticsearch.indices.InvalidIndexNameException;
|
||||
import org.elasticsearch.search.SearchHit;
|
||||
import org.elasticsearch.test.ESIntegTestCase;
|
||||
|
||||
|
|
@ -296,22 +297,15 @@ public class SimpleIndexTemplateIT extends ESIntegTestCase {
|
|||
GetIndexTemplatesResponse response = client().admin().indices().prepareGetTemplates().get();
|
||||
assertThat(response.getIndexTemplates(), empty());
|
||||
|
||||
client().admin().indices().preparePutTemplate("template_1")
|
||||
MapperParsingException e = expectThrows( MapperParsingException.class,
|
||||
() -> client().admin().indices().preparePutTemplate("template_1")
|
||||
.setTemplate("te*")
|
||||
.addMapping("type1", "abcde")
|
||||
.get();
|
||||
.get());
|
||||
assertThat(e.getMessage(), containsString("Failed to parse mapping "));
|
||||
|
||||
response = client().admin().indices().prepareGetTemplates().get();
|
||||
assertThat(response.getIndexTemplates(), hasSize(1));
|
||||
assertThat(response.getIndexTemplates().get(0).getMappings().size(), equalTo(1));
|
||||
assertThat(response.getIndexTemplates().get(0).getMappings().get("type1").string(), equalTo("abcde"));
|
||||
|
||||
try {
|
||||
createIndex("test");
|
||||
fail("create index should have failed due to broken index templates mapping");
|
||||
} catch(ElasticsearchParseException e) {
|
||||
//everything fine
|
||||
}
|
||||
assertThat(response.getIndexTemplates(), hasSize(0));
|
||||
}
|
||||
|
||||
public void testInvalidSettings() throws Exception {
|
||||
|
|
@ -322,15 +316,12 @@ public class SimpleIndexTemplateIT extends ESIntegTestCase {
|
|||
GetIndexTemplatesResponse response = client().admin().indices().prepareGetTemplates().get();
|
||||
assertThat(response.getIndexTemplates(), empty());
|
||||
|
||||
try {
|
||||
client().admin().indices().preparePutTemplate("template_1")
|
||||
IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
|
||||
() -> client().admin().indices().preparePutTemplate("template_1")
|
||||
.setTemplate("te*")
|
||||
.setSettings(Settings.builder().put("does_not_exist", "test"))
|
||||
.get();
|
||||
fail();
|
||||
} catch (IllegalArgumentException ex) {
|
||||
assertEquals("unknown setting [index.does_not_exist]", ex.getMessage());
|
||||
}
|
||||
.get());
|
||||
assertEquals("unknown setting [index.does_not_exist]", e.getMessage());
|
||||
|
||||
response = client().admin().indices().prepareGetTemplates().get();
|
||||
assertEquals(0, response.getIndexTemplates().size());
|
||||
|
|
@ -490,15 +481,12 @@ public class SimpleIndexTemplateIT extends ESIntegTestCase {
|
|||
assertThat(response.getIndexTemplates().get(0).getAliases().size(), equalTo(1));
|
||||
assertThat(response.getIndexTemplates().get(0).getAliases().get("invalid_alias").filter().string(), equalTo("{\"invalid\":{}}"));
|
||||
|
||||
try {
|
||||
createIndex("test");
|
||||
fail("index creation should have failed due to invalid alias filter in matching index template");
|
||||
} catch(IllegalArgumentException e) {
|
||||
IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
|
||||
() -> createIndex("test"));
|
||||
assertThat(e.getMessage(), equalTo("failed to parse filter for alias [invalid_alias]"));
|
||||
assertThat(e.getCause(), instanceOf(ParsingException.class));
|
||||
assertThat(e.getCause().getMessage(), equalTo("no [query] registered for [invalid]"));
|
||||
}
|
||||
}
|
||||
|
||||
public void testAliasInvalidFilterInvalidJson() throws Exception {
|
||||
//invalid json: put index template fails
|
||||
|
|
@ -506,11 +494,9 @@ public class SimpleIndexTemplateIT extends ESIntegTestCase {
|
|||
.setTemplate("te*")
|
||||
.addAlias(new Alias("invalid_alias").filter("abcde"));
|
||||
|
||||
try {
|
||||
putIndexTemplateRequestBuilder.get();
|
||||
} catch(IllegalArgumentException e) {
|
||||
IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
|
||||
() -> putIndexTemplateRequestBuilder.get());
|
||||
assertThat(e.getMessage(), equalTo("failed to parse filter for alias [invalid_alias]"));
|
||||
}
|
||||
|
||||
GetIndexTemplatesResponse response = client().admin().indices().prepareGetTemplates("template_1").get();
|
||||
assertThat(response.getIndexTemplates().size(), equalTo(0));
|
||||
|
|
@ -523,39 +509,30 @@ public class SimpleIndexTemplateIT extends ESIntegTestCase {
|
|||
.setTemplate("te*")
|
||||
.addAlias(new Alias("index")).get();
|
||||
|
||||
try {
|
||||
createIndex("test");
|
||||
fail("index creation should have failed due to alias with existing index name in matching index template");
|
||||
} catch(InvalidAliasNameException e) {
|
||||
InvalidAliasNameException e = expectThrows(InvalidAliasNameException.class,
|
||||
() -> createIndex("test"));
|
||||
assertThat(e.getMessage(), equalTo("Invalid alias name [index], an index exists with the same name as the alias"));
|
||||
}
|
||||
}
|
||||
|
||||
public void testAliasEmptyName() throws Exception {
|
||||
PutIndexTemplateRequestBuilder putIndexTemplateRequestBuilder = client().admin().indices().preparePutTemplate("template_1")
|
||||
.setTemplate("te*")
|
||||
.addAlias(new Alias(" ").indexRouting("1,2,3"));
|
||||
|
||||
try {
|
||||
putIndexTemplateRequestBuilder.get();
|
||||
fail("put template should have failed due to alias with empty name");
|
||||
} catch (IllegalArgumentException e) {
|
||||
IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
|
||||
() -> putIndexTemplateRequestBuilder.get());
|
||||
assertThat(e.getMessage(), equalTo("alias name is required"));
|
||||
}
|
||||
}
|
||||
|
||||
public void testAliasWithMultipleIndexRoutings() throws Exception {
|
||||
PutIndexTemplateRequestBuilder putIndexTemplateRequestBuilder = client().admin().indices().preparePutTemplate("template_1")
|
||||
.setTemplate("te*")
|
||||
.addAlias(new Alias("alias").indexRouting("1,2,3"));
|
||||
|
||||
try {
|
||||
putIndexTemplateRequestBuilder.get();
|
||||
fail("put template should have failed due to alias with multiple index routings");
|
||||
} catch (IllegalArgumentException e) {
|
||||
IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
|
||||
() -> putIndexTemplateRequestBuilder.get());
|
||||
assertThat(e.getMessage(), equalTo("alias [alias] has several index routing values associated with it"));
|
||||
}
|
||||
}
|
||||
|
||||
public void testMultipleAliasesPrecedence() throws Exception {
|
||||
client().admin().indices().preparePutTemplate("template1")
|
||||
|
|
@ -664,4 +641,48 @@ public class SimpleIndexTemplateIT extends ESIntegTestCase {
|
|||
assertThat(response.getItems()[0].getVersion(), equalTo(1L));
|
||||
}
|
||||
|
||||
public void testCombineTemplates() throws Exception{
|
||||
// clean all templates setup by the framework.
|
||||
client().admin().indices().prepareDeleteTemplate("*").get();
|
||||
|
||||
// check get all templates on an empty index.
|
||||
GetIndexTemplatesResponse response = client().admin().indices().prepareGetTemplates().get();
|
||||
assertThat(response.getIndexTemplates(), empty());
|
||||
|
||||
//Now, a complete mapping with two separated templates is error
|
||||
// base template
|
||||
client().admin().indices().preparePutTemplate("template_1")
|
||||
.setTemplate("*")
|
||||
.setSettings(
|
||||
" {\n" +
|
||||
" \"index\" : {\n" +
|
||||
" \"analysis\" : {\n" +
|
||||
" \"analyzer\" : {\n" +
|
||||
" \"custom_1\" : {\n" +
|
||||
" \"tokenizer\" : \"whitespace\"\n" +
|
||||
" }\n" +
|
||||
" }\n" +
|
||||
" }\n" +
|
||||
" }\n" +
|
||||
" }\n")
|
||||
.get();
|
||||
|
||||
// put template using custom_1 analyzer
|
||||
MapperParsingException e = expectThrows(MapperParsingException.class,
|
||||
() -> client().admin().indices().preparePutTemplate("template_2")
|
||||
.setTemplate("test*")
|
||||
.setCreate(true)
|
||||
.setOrder(1)
|
||||
.addMapping("type1", XContentFactory.jsonBuilder().startObject().startObject("type1").startObject("properties")
|
||||
.startObject("field2").field("type", "string").field("analyzer", "custom_1").endObject()
|
||||
.endObject().endObject().endObject())
|
||||
.get());
|
||||
assertThat(e.getMessage(), containsString("analyzer [custom_1] not found for field [field2]"));
|
||||
|
||||
response = client().admin().indices().prepareGetTemplates().get();
|
||||
assertThat(response.getIndexTemplates(), hasSize(1));
|
||||
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -59,6 +59,7 @@ import org.elasticsearch.common.unit.ByteSizeUnit;
|
|||
import org.elasticsearch.common.unit.TimeValue;
|
||||
import org.elasticsearch.index.shard.ShardId;
|
||||
import org.elasticsearch.index.store.IndexStore;
|
||||
import org.elasticsearch.common.xcontent.XContentFactory;
|
||||
import org.elasticsearch.indices.InvalidIndexNameException;
|
||||
import org.elasticsearch.repositories.RepositoriesService;
|
||||
import org.elasticsearch.repositories.RepositoryException;
|
||||
|
|
@ -416,7 +417,10 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
|
|||
.setType("fs").setSettings(Settings.builder().put("location", randomRepoPath())));
|
||||
|
||||
logger.info("--> creating test template");
|
||||
assertThat(client.admin().indices().preparePutTemplate("test-template").setTemplate("te*").addMapping("test-mapping", "{}").get().isAcknowledged(), equalTo(true));
|
||||
assertThat(client.admin().indices().preparePutTemplate("test-template").setTemplate("te*").addMapping("test-mapping", XContentFactory.jsonBuilder().startObject().startObject("test-mapping").startObject("properties")
|
||||
.startObject("field1").field("type", "string").field("store", "yes").endObject()
|
||||
.startObject("field2").field("type", "string").field("store", "yes").field("index", "not_analyzed").endObject()
|
||||
.endObject().endObject().endObject()).get().isAcknowledged(), equalTo(true));
|
||||
|
||||
logger.info("--> snapshot");
|
||||
CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap").setIndices().setWaitForCompletion(true).get();
|
||||
|
|
@ -449,7 +453,10 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
|
|||
.setType("fs").setSettings(Settings.builder().put("location", location)));
|
||||
|
||||
logger.info("--> creating test template");
|
||||
assertThat(client.admin().indices().preparePutTemplate("test-template").setTemplate("te*").addMapping("test-mapping", "{}").get().isAcknowledged(), equalTo(true));
|
||||
assertThat(client.admin().indices().preparePutTemplate("test-template").setTemplate("te*").addMapping("test-mapping", XContentFactory.jsonBuilder().startObject().startObject("test-mapping").startObject("properties")
|
||||
.startObject("field1").field("type", "string").field("store", "yes").endObject()
|
||||
.startObject("field2").field("type", "string").field("store", "yes").field("index", "not_analyzed").endObject()
|
||||
.endObject().endObject().endObject()).get().isAcknowledged(), equalTo(true));
|
||||
|
||||
logger.info("--> snapshot without global state");
|
||||
CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap-no-global-state").setIndices().setIncludeGlobalState(false).setWaitForCompletion(true).get();
|
||||
|
|
|
|||
Loading…
Reference in New Issue