honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Update IndexTemplateMetaData to allow unknown fields (#38448) 

Updated IndexTemplateMetaData to use ObjectParser.

The IndexTemplateMetaData class used old parsing logic and was not
resiliant to new fields. This commit updates it to use the
ConstructingObjectParser and allow unknown fields.

Relates #36938

Co-authored-by: Michael Basnight <mbasnight@gmail.com>
Co-authored-by: Martijn van Groningen <martijn.v.groningen@gmail.com>
This commit is contained in:
Martijn van Groningen 2019-02-06 08:21:30 +01:00 committed by GitHub
parent e1d464b22c
commit 9492dcea42
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 184 additions and 95 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

142
client/rest-high-level/src/main/java/org/elasticsearch/client/indices/IndexTemplateMetaData.java
View File

@ -18,27 +18,66 @@
*/
package org.elasticsearch.client.indices;
import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.cluster.metadata.AliasMetaData;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.metadata.MappingMetaData;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.set.Sets;
import org.elasticsearch.common.xcontent.ConstructingObjectParser;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.mapper.MapperService;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.AbstractMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;
public class IndexTemplateMetaData {
@SuppressWarnings("unchecked")
private static final ConstructingObjectParser<IndexTemplateMetaData, String> PARSER = new ConstructingObjectParser<>(
"IndexTemplateMetaData", true, (a, name) -> {
List<Map.Entry<String, AliasMetaData>> alias = (List<Map.Entry<String, AliasMetaData>>) a[5];
ImmutableOpenMap<String, AliasMetaData> aliasMap =
new ImmutableOpenMap.Builder<String, AliasMetaData>()
.putAll(alias.stream().collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)))
.build();
return new IndexTemplateMetaData(
name,
(Integer) a[0],
(Integer) a[1],
(List<String>) a[2],
(Settings) a[3],
(MappingMetaData) a[4],
aliasMap);
});
static {
PARSER.declareInt(optionalConstructorArg(), new ParseField("order"));
PARSER.declareInt(optionalConstructorArg(), new ParseField("version"));
PARSER.declareStringArray(optionalConstructorArg(), new ParseField("index_patterns"));
PARSER.declareObject(optionalConstructorArg(), (p, c) -> {
Settings.Builder templateSettingsBuilder = Settings.builder();
templateSettingsBuilder.put(Settings.fromXContent(p));
templateSettingsBuilder.normalizePrefix(IndexMetaData.INDEX_SETTING_PREFIX);
return templateSettingsBuilder.build();
}, new ParseField("settings"));
PARSER.declareObject(optionalConstructorArg(), (p, c) -> {
Map<String, Object> mapping = p.map();
if (mapping.isEmpty()) {
return null;
}
return new MappingMetaData(MapperService.SINGLE_MAPPING_NAME, mapping);
}, new ParseField("mappings"));
PARSER.declareNamedObjects(optionalConstructorArg(),
(p, c, name) -> new AbstractMap.SimpleEntry<>(name, AliasMetaData.Builder.fromXContent(p)), new ParseField("aliases"));
}
private final String name;
@ -125,28 +164,23 @@ public class IndexTemplateMetaData {
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
IndexTemplateMetaData that = (IndexTemplateMetaData) o;
if (order != that.order) return false;
if (!Objects.equals(mappings, that.mappings)) return false;
if (!name.equals(that.name)) return false;
if (!settings.equals(that.settings)) return false;
if (!patterns.equals(that.patterns)) return false;
return Objects.equals(version, that.version);
return order == that.order &&
Objects.equals(name, that.name) &&
Objects.equals(version, that.version) &&
Objects.equals(patterns, that.patterns) &&
Objects.equals(settings, that.settings) &&
Objects.equals(mappings, that.mappings) &&
Objects.equals(aliases, that.aliases);
}
@Override
public int hashCode() {
return Objects.hash(name, order, version, patterns, settings, mappings);
return Objects.hash(name, order, version, patterns, settings, mappings, aliases);
}
public static class Builder {
private static final Set<String> VALID_FIELDS = Sets.newHashSet(
"template", "order", "mappings", "settings", "index_patterns", "aliases", "version");
private String name;
private int order;
@ -193,7 +227,6 @@ public class IndexTemplateMetaData {
return this;
}
public Builder settings(Settings.Builder settings) {
this.settings = settings.build();
return this;
@ -225,76 +258,7 @@ public class IndexTemplateMetaData {
public static IndexTemplateMetaData fromXContent(XContentParser parser, String templateName) throws IOException {
Builder builder = new Builder(templateName);
String currentFieldName = skipTemplateName(parser);
XContentParser.Token token;
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
currentFieldName = parser.currentName();
} else if (token == XContentParser.Token.START_OBJECT) {
if ("settings".equals(currentFieldName)) {
Settings.Builder templateSettingsBuilder = Settings.builder();
templateSettingsBuilder.put(Settings.fromXContent(parser));
templateSettingsBuilder.normalizePrefix(IndexMetaData.INDEX_SETTING_PREFIX);
builder.settings(templateSettingsBuilder.build());
} else if ("mappings".equals(currentFieldName)) {
Map<String, Object> mapping = parser.map();
if (mapping.isEmpty() == false) {
MappingMetaData md = new MappingMetaData(MapperService.SINGLE_MAPPING_NAME, mapping);
builder.mapping(md);
}
} else if ("aliases".equals(currentFieldName)) {
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
builder.putAlias(AliasMetaData.Builder.fromXContent(parser));
}
} else {
throw new ElasticsearchParseException("unknown key [{}] for index template", currentFieldName);
}
} else if (token == XContentParser.Token.START_ARRAY) {
if ("mappings".equals(currentFieldName)) {
// The server-side IndexTemplateMetaData has toXContent impl that can return mappings
// in an array but also a comment saying this never happens with typeless APIs.
throw new ElasticsearchParseException("Invalid response format - "
+ "mappings are not expected to be returned in an array", currentFieldName);
} else if ("index_patterns".equals(currentFieldName)) {
List<String> index_patterns = new ArrayList<>();
while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
index_patterns.add(parser.text());
}
builder.patterns(index_patterns);
}
} else if (token.isValue()) {
// Prior to 5.1.0, elasticsearch only supported a single index pattern called `template` (#21009)
if("template".equals(currentFieldName)) {
builder.patterns(Collections.singletonList(parser.text()));
} else if ("order".equals(currentFieldName)) {
builder.order(parser.intValue());
} else if ("version".equals(currentFieldName)) {
builder.version(parser.intValue());
}
}
}
return builder.build();
}
private static String skipTemplateName(XContentParser parser) throws IOException {
XContentParser.Token token = parser.nextToken();
if (token == XContentParser.Token.START_OBJECT) {
token = parser.nextToken();
if (token == XContentParser.Token.FIELD_NAME) {
String currentFieldName = parser.currentName();
if (VALID_FIELDS.contains(currentFieldName)) {
return currentFieldName;
} else {
// we just hit the template name, which should be ignored and we move on
parser.nextToken();
}
}
}
return null;
return PARSER.parse(parser, templateName);
}
}
}

137
client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetIndexTemplatesResponseTests.java
View File

@ -22,10 +22,16 @@ package org.elasticsearch.client.indices;
import org.elasticsearch.cluster.metadata.AliasMetaData;
import org.elasticsearch.cluster.metadata.MappingMetaData;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.compress.CompressedXContent;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.DeprecationHandler;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.test.ESTestCase;
@ -33,13 +39,20 @@ import org.elasticsearch.test.ESTestCase;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
import java.util.Iterator;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.function.Predicate;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import static org.elasticsearch.index.RandomCreateIndexGenerator.randomIndexSettings;
import static org.elasticsearch.index.RandomCreateIndexGenerator.randomMappingFields;
import static org.elasticsearch.test.AbstractXContentTestCase.xContentTester;
import static org.hamcrest.Matchers.equalTo;
public class GetIndexTemplatesResponseTests extends ESTestCase {
@ -50,11 +63,88 @@ public class GetIndexTemplatesResponseTests extends ESTestCase {
public void testFromXContent() throws IOException {
xContentTester(this::createParser, GetIndexTemplatesResponseTests::createTestInstance, GetIndexTemplatesResponseTests::toXContent,
GetIndexTemplatesResponse::fromXContent).supportsUnknownFields(false)
.assertEqualsConsumer(GetIndexTemplatesResponseTests::assertEqualInstances)
.shuffleFieldsExceptions(new String[] {"aliases", "mappings", "patterns", "settings"})
.test();
xContentTester(this::createParser,
GetIndexTemplatesResponseTests::createTestInstance,
GetIndexTemplatesResponseTests::toXContent,
GetIndexTemplatesResponse::fromXContent)
.assertEqualsConsumer(GetIndexTemplatesResponseTests::assertEqualInstances)
.supportsUnknownFields(true)
.randomFieldsExcludeFilter(randomFieldsExcludeFilter())
.shuffleFieldsExceptions(new String[] {"aliases", "mappings", "patterns", "settings"})
.test();
}
public void testParsingFromEsResponse() throws IOException {
for (int runs = 0; runs < 20; runs++) {
org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse esResponse =
new org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse(new ArrayList<>());
XContentType xContentType = randomFrom(XContentType.values());
int numTemplates = randomIntBetween(0, 32);
for (int i = 0; i < numTemplates; i++) {
org.elasticsearch.cluster.metadata.IndexTemplateMetaData.Builder esIMD =
new org.elasticsearch.cluster.metadata.IndexTemplateMetaData.Builder(String.format(Locale.ROOT, "%02d ", i) +
randomAlphaOfLength(4));
esIMD.patterns(Arrays.asList(generateRandomStringArray(32, 4, false, false)));
esIMD.settings(randomIndexSettings());
esIMD.putMapping("_doc", new CompressedXContent(BytesReference.bytes(randomMapping("_doc", xContentType))));
int numAliases = randomIntBetween(0, 8);
for (int j = 0; j < numAliases; j++) {
esIMD.putAlias(randomAliasMetaData(String.format(Locale.ROOT, "%02d ", j) + randomAlphaOfLength(4)));
}
esIMD.order(randomIntBetween(0, Integer.MAX_VALUE));
esIMD.version(randomIntBetween(0, Integer.MAX_VALUE));
esResponse.getIndexTemplates().add(esIMD.build());
}
XContentBuilder xContentBuilder = XContentBuilder.builder(xContentType.xContent());
esResponse.toXContent(xContentBuilder, ToXContent.EMPTY_PARAMS);
try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY,
DeprecationHandler.THROW_UNSUPPORTED_OPERATION, BytesReference.bytes(xContentBuilder), xContentType)) {
GetIndexTemplatesResponse response = GetIndexTemplatesResponse.fromXContent(parser);
assertThat(response.getIndexTemplates().size(), equalTo(numTemplates));
response.getIndexTemplates().sort(Comparator.comparing(IndexTemplateMetaData::name));
for (int i = 0; i < numTemplates; i++) {
org.elasticsearch.cluster.metadata.IndexTemplateMetaData esIMD = esResponse.getIndexTemplates().get(i);
IndexTemplateMetaData result = response.getIndexTemplates().get(i);
assertThat(result.patterns(), equalTo(esIMD.patterns()));
assertThat(result.settings(), equalTo(esIMD.settings()));
assertThat(result.order(), equalTo(esIMD.order()));
assertThat(result.version(), equalTo(esIMD.version()));
assertThat(esIMD.mappings().size(), equalTo(1));
BytesArray mappingSource = new BytesArray(esIMD.mappings().valuesIt().next().uncompressed());
Map<String, Object> expectedMapping =
XContentHelper.convertToMap(mappingSource, true, xContentBuilder.contentType()).v2();
assertThat(result.mappings().sourceAsMap(), equalTo(expectedMapping.get("_doc")));
assertThat(result.aliases().size(), equalTo(esIMD.aliases().size()));
List<AliasMetaData> expectedAliases = Arrays.stream(esIMD.aliases().values().toArray(AliasMetaData.class))
.sorted(Comparator.comparing(AliasMetaData::alias))
.collect(Collectors.toList());
List<AliasMetaData> actualAliases = Arrays.stream(result.aliases().values().toArray(AliasMetaData.class))
.sorted(Comparator.comparing(AliasMetaData::alias))
.collect(Collectors.toList());
for (int j = 0; j < result.aliases().size(); j++) {
assertThat(actualAliases.get(j), equalTo(expectedAliases.get(j)));
}
}
}
}
}
private Predicate<String> randomFieldsExcludeFilter() {
return (field) ->
field.isEmpty()
|| field.endsWith("aliases")
|| field.endsWith("settings")
|| field.endsWith("settings.index")
|| field.endsWith("mappings") // uses parser.map()
|| field.contains("mappings.properties") // cannot have extra properties
;
}
private static void assertEqualInstances(GetIndexTemplatesResponse expectedInstance, GetIndexTemplatesResponse newInstance) {
@ -64,7 +154,7 @@ public class GetIndexTemplatesResponseTests extends ESTestCase {
new BytesArray(mappingString), true, XContentType.JSON).v2();
for (IndexTemplateMetaData template : newInstance.getIndexTemplates()) {
MappingMetaData mappingMD = template.mappings();
if(mappingMD!=null) {
if(mappingMD != null) {
Map<String, Object> mappingAsMap = mappingMD.sourceAsMap();
assertEquals(expectedMap, mappingAsMap);
}
@ -133,4 +223,39 @@ public class GetIndexTemplatesResponseTests extends ESTestCase {
org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse(serverIndexTemplates);
serverResponse.toXContent(builder, ToXContent.EMPTY_PARAMS);
}
private static AliasMetaData randomAliasMetaData(String name) {
AliasMetaData.Builder alias = AliasMetaData.builder(name);
if (randomBoolean()) {
if (randomBoolean()) {
alias.routing(randomAlphaOfLength(5));
} else {
if (randomBoolean()) {
alias.indexRouting(randomAlphaOfLength(5));
}
if (randomBoolean()) {
alias.searchRouting(randomAlphaOfLength(5));
}
}
}
if (randomBoolean()) {
alias.filter("{\"term\":{\"year\":2016}}");
}
if (randomBoolean()) {
alias.writeIndex(randomBoolean());
}
return alias.build();
}
static XContentBuilder randomMapping(String type, XContentType xContentType) throws IOException {
XContentBuilder builder = XContentFactory.contentBuilder(xContentType);
builder.startObject().startObject(type);
randomMappingFields(builder, true);
builder.endObject().endObject();
return builder;
}
}