diff --git a/core/src/main/java/org/elasticsearch/index/mapper/Mapper.java b/core/src/main/java/org/elasticsearch/index/mapper/Mapper.java index a8d0c0a706d..c9877410c30 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/Mapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/Mapper.java @@ -134,6 +134,26 @@ public abstract class Mapper implements ToXContent, Iterable { public ParseFieldMatcher parseFieldMatcher() { return parseFieldMatcher; } + + public boolean isWithinMultiField() { return false; } + + protected Function typeParsers() { return typeParsers; } + + protected Function similarityLookupService() { return similarityLookupService; } + + public ParserContext createMultiFieldContext(ParserContext in) { + return new MultiFieldParserContext(in) { + @Override + public boolean isWithinMultiField() { return true; } + }; + } + + class MultiFieldParserContext extends ParserContext { + MultiFieldParserContext(ParserContext in) { + super(in.type(), in.analysisService, in.similarityLookupService(), in.mapperService(), in.typeParsers(), in.indexVersionCreated(), in.parseFieldMatcher()); + } + } + } Mapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException; diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/TypeParsers.java b/core/src/main/java/org/elasticsearch/index/mapper/core/TypeParsers.java index c0e0cef0f0c..a3938a48a5b 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/core/TypeParsers.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/core/TypeParsers.java @@ -25,6 +25,7 @@ import org.elasticsearch.Version; import org.elasticsearch.common.Strings; import org.elasticsearch.common.joda.FormatDateTimeFormatter; import org.elasticsearch.common.joda.Joda; +import org.elasticsearch.common.logging.ESLoggerFactory; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.loader.SettingsLoader; import org.elasticsearch.index.analysis.NamedAnalyzer; @@ -184,11 +185,12 @@ public class TypeParsers { public static void parseField(FieldMapper.Builder builder, String name, Map fieldNode, Mapper.TypeParser.ParserContext parserContext) { NamedAnalyzer indexAnalyzer = builder.fieldType().indexAnalyzer(); NamedAnalyzer searchAnalyzer = builder.fieldType().searchAnalyzer(); + Version indexVersionCreated = parserContext.indexVersionCreated(); for (Iterator> iterator = fieldNode.entrySet().iterator(); iterator.hasNext();) { Map.Entry entry = iterator.next(); final String propName = Strings.toUnderscoreCase(entry.getKey()); final Object propNode = entry.getValue(); - if (propName.equals("index_name") && parserContext.indexVersionCreated().before(Version.V_2_0_0_beta1)) { + if (propName.equals("index_name") && indexVersionCreated.before(Version.V_2_0_0_beta1)) { builder.indexName(propNode.toString()); iterator.remove(); } else if (propName.equals("store")) { @@ -239,7 +241,7 @@ public class TypeParsers { iterator.remove(); } else if (propName.equals("omit_term_freq_and_positions")) { final IndexOptions op = nodeBooleanValue(propNode) ? IndexOptions.DOCS : IndexOptions.DOCS_AND_FREQS_AND_POSITIONS; - if (parserContext.indexVersionCreated().onOrAfter(Version.V_1_0_0_RC2)) { + if (indexVersionCreated.onOrAfter(Version.V_1_0_0_RC2)) { throw new ElasticsearchParseException("'omit_term_freq_and_positions' is not supported anymore - use ['index_options' : 'docs'] instead"); } // deprecated option for BW compat @@ -249,8 +251,8 @@ public class TypeParsers { builder.indexOptions(nodeIndexOptionValue(propNode)); iterator.remove(); } else if (propName.equals("analyzer") || // for backcompat, reading old indexes, remove for v3.0 - propName.equals("index_analyzer") && parserContext.indexVersionCreated().before(Version.V_2_0_0_beta1)) { - + propName.equals("index_analyzer") && indexVersionCreated.before(Version.V_2_0_0_beta1)) { + NamedAnalyzer analyzer = parserContext.analysisService().analyzer(propNode.toString()); if (analyzer == null) { throw new MapperParsingException("analyzer [" + propNode.toString() + "] not found for field [" + name + "]"); @@ -267,10 +269,10 @@ public class TypeParsers { } else if (propName.equals("include_in_all")) { builder.includeInAll(nodeBooleanValue(propNode)); iterator.remove(); - } else if (propName.equals("postings_format") && parserContext.indexVersionCreated().before(Version.V_2_0_0_beta1)) { + } else if (propName.equals("postings_format") && indexVersionCreated.before(Version.V_2_0_0_beta1)) { // ignore for old indexes iterator.remove(); - } else if (propName.equals("doc_values_format") && parserContext.indexVersionCreated().before(Version.V_2_0_0_beta1)) { + } else if (propName.equals("doc_values_format") && indexVersionCreated.before(Version.V_2_0_0_beta1)) { // ignore for old indexes iterator.remove(); } else if (propName.equals("similarity")) { @@ -281,7 +283,16 @@ public class TypeParsers { builder.fieldDataSettings(settings); iterator.remove(); } else if (propName.equals("copy_to")) { - parseCopyFields(propNode, builder); + if (parserContext.isWithinMultiField()) { + if (indexVersionCreated.after(Version.V_2_1_0) || + (indexVersionCreated.after(Version.V_2_0_1) && indexVersionCreated.before(Version.V_2_1_0))) { + throw new MapperParsingException("copy_to in multi fields is not allowed. Found the copy_to in field [" + name + "] which is within a multi field."); + } else { + ESLoggerFactory.getLogger("mapping [" + parserContext.type() + "]").warn("Found a copy_to in field [" + name + "] which is within a multi field. This feature has been removed and the copy_to will be removed from the mapping."); + } + } else { + parseCopyFields(propNode, builder); + } iterator.remove(); } } @@ -298,6 +309,7 @@ public class TypeParsers { } public static boolean parseMultiField(FieldMapper.Builder builder, String name, Mapper.TypeParser.ParserContext parserContext, String propName, Object propNode) { + parserContext = parserContext.createMultiFieldContext(parserContext); if (propName.equals("path") && parserContext.indexVersionCreated().before(Version.V_2_0_0_beta1)) { builder.multiFieldPathType(parsePathType(name, propNode.toString())); return true; diff --git a/core/src/test/java/org/elasticsearch/index/mapper/core/MultiFieldCopyToMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/core/MultiFieldCopyToMapperTests.java new file mode 100644 index 00000000000..821eaeb8365 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/index/mapper/core/MultiFieldCopyToMapperTests.java @@ -0,0 +1,105 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +package org.elasticsearch.index.mapper.core; + +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.common.collect.Tuple; +import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.index.MapperTestUtils; +import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.MapperParsingException; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.VersionUtils; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.hamcrest.core.IsEqual.equalTo; + +public class MultiFieldCopyToMapperTests extends ESTestCase { + + public void testExceptionForCopyToInMultiFields() throws IOException { + XContentBuilder mapping = createMappinmgWithCopyToInMultiField(); + Tuple, List> versionsWithAndWithoutExpectedExceptions = versionsWithAndWithoutExpectedExceptions(); + + // first check that for newer versions we throw exception if copy_to is found withing multi field + Version indexVersion = randomFrom(versionsWithAndWithoutExpectedExceptions.v1()); + MapperService mapperService = MapperTestUtils.newMapperService(createTempDir(), Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, indexVersion).build()); + try { + mapperService.parse("type", new CompressedXContent(mapping.string()), true); + fail("Parsing should throw an exception because the mapping contains a copy_to in a multi field"); + } catch (MapperParsingException e) { + assertThat(e.getMessage(), equalTo("copy_to in multi fields is not allowed. Found the copy_to in field [c] which is within a multi field.")); + } + + // now test that with an older version the pasring just works + indexVersion = randomFrom(versionsWithAndWithoutExpectedExceptions.v2()); + mapperService = MapperTestUtils.newMapperService(createTempDir(), Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, indexVersion).build()); + DocumentMapper documentMapper = mapperService.parse("type", new CompressedXContent(mapping.string()), true); + assertFalse(documentMapper.mapping().toString().contains("copy_to")); + } + + private static XContentBuilder createMappinmgWithCopyToInMultiField() throws IOException { + XContentBuilder mapping = jsonBuilder(); + mapping.startObject() + .startObject("type") + .startObject("properties") + .startObject("a") + .field("type", "string") + .endObject() + .startObject("b") + .field("type", "string") + .startObject("fields") + .startObject("c") + .field("type", "string") + .field("copy_to", "a") + .endObject() + .endObject() + .endObject() + .endObject() + .endObject() + .endObject(); + return mapping; + } + + // returs a tuple where + // v1 is a list of versions for which we expect an excpetion when a copy_to in multi fields is found and + // v2 is older versions where we throw no exception and we just log a warning + private static Tuple, List> versionsWithAndWithoutExpectedExceptions() { + List versionsWithException = new ArrayList<>(); + List versionsWithoutException = new ArrayList<>(); + for (Version version : VersionUtils.allVersions()) { + if (version.after(Version.V_2_1_0) || + (version.after(Version.V_2_0_1) && version.before(Version.V_2_1_0))) { + versionsWithException.add(version); + } else { + versionsWithoutException.add(version); + } + } + return new Tuple<>(versionsWithException, versionsWithoutException); + } +} diff --git a/docs/reference/migration/migrate_2_0/mapping.asciidoc b/docs/reference/migration/migrate_2_0/mapping.asciidoc index 33ef9ebc5e5..09170a2e718 100644 --- a/docs/reference/migration/migrate_2_0/mapping.asciidoc +++ b/docs/reference/migration/migrate_2_0/mapping.asciidoc @@ -429,3 +429,11 @@ to use the old default of 0. This was done to prevent phrase queries from matching across different values of the same term unexpectedly. Specifically, 100 was chosen to cause phrase queries with slops up to 99 to match only within a single value of a field. + +==== copy_to and multi fields + +A <> within a <> is ignored from version 2.0 on. With any version after +2.1 or 2.0.1 creating a mapping that has a copy_to within a multi field will result +in an exception. + + diff --git a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/AttachmentUnitTestCase.java b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/AttachmentUnitTestCase.java index 9378f2d71f9..9b7d8afe381 100644 --- a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/AttachmentUnitTestCase.java +++ b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/AttachmentUnitTestCase.java @@ -22,13 +22,20 @@ package org.elasticsearch.mapper.attachments; import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.indices.IndicesModule; import org.elasticsearch.test.ESTestCase; import org.junit.Before; public class AttachmentUnitTestCase extends ESTestCase { protected Settings testSettings; - + + protected static IndicesModule getIndicesModuleWithRegisteredAttachmentMapper() { + IndicesModule indicesModule = new IndicesModule(); + indicesModule.registerMapper(AttachmentMapper.CONTENT_TYPE, new AttachmentMapper.TypeParser()); + return indicesModule; + } + @Before public void createSettings() throws Exception { testSettings = Settings.builder() diff --git a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/DateAttachmentMapperTests.java b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/DateAttachmentMapperTests.java index 858ed8a767f..f93785ed14a 100644 --- a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/DateAttachmentMapperTests.java +++ b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/DateAttachmentMapperTests.java @@ -20,6 +20,7 @@ package org.elasticsearch.mapper.attachments; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.MapperTestUtils; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.DocumentMapperParser; import org.elasticsearch.index.mapper.core.StringFieldMapper; @@ -37,7 +38,7 @@ public class DateAttachmentMapperTests extends AttachmentUnitTestCase { @Before public void setupMapperParser() throws Exception { - mapperParser = MapperTestUtils.newMapperService(createTempDir(), Settings.EMPTY).documentMapperParser(); + mapperParser = MapperTestUtils.newMapperService(createTempDir(), Settings.EMPTY, getIndicesModuleWithRegisteredAttachmentMapper()).documentMapperParser(); } public void testSimpleMappings() throws Exception { diff --git a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/EncryptedDocMapperTests.java b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/EncryptedDocMapperTests.java index e086d9ba5c4..10e82e24c84 100644 --- a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/EncryptedDocMapperTests.java +++ b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/EncryptedDocMapperTests.java @@ -21,11 +21,11 @@ package org.elasticsearch.mapper.attachments; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.MapperTestUtils; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.DocumentMapperParser; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.ParseContext; -import org.elasticsearch.mapper.attachments.AttachmentMapper; import java.io.IOException; @@ -42,7 +42,7 @@ import static org.hamcrest.Matchers.*; public class EncryptedDocMapperTests extends AttachmentUnitTestCase { public void testMultipleDocsEncryptedLast() throws IOException { - DocumentMapperParser mapperParser = MapperTestUtils.newMapperService(createTempDir(), Settings.EMPTY).documentMapperParser(); + DocumentMapperParser mapperParser = MapperTestUtils.newMapperService(createTempDir(), Settings.EMPTY, getIndicesModuleWithRegisteredAttachmentMapper()).documentMapperParser(); String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/attachment/test/unit/encrypted/test-mapping.json"); DocumentMapper docMapper = mapperParser.parse(mapping); @@ -72,7 +72,7 @@ public class EncryptedDocMapperTests extends AttachmentUnitTestCase { } public void testMultipleDocsEncryptedFirst() throws IOException { - DocumentMapperParser mapperParser = MapperTestUtils.newMapperService(createTempDir(), Settings.EMPTY).documentMapperParser(); + DocumentMapperParser mapperParser = MapperTestUtils.newMapperService(createTempDir(), Settings.EMPTY, getIndicesModuleWithRegisteredAttachmentMapper()).documentMapperParser(); String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/attachment/test/unit/encrypted/test-mapping.json"); DocumentMapper docMapper = mapperParser.parse(mapping); byte[] html = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/attachment/test/sample-files/htmlWithValidDateMeta.html"); @@ -103,9 +103,8 @@ public class EncryptedDocMapperTests extends AttachmentUnitTestCase { public void testMultipleDocsEncryptedNotIgnoringErrors() throws IOException { try { DocumentMapperParser mapperParser = MapperTestUtils.newMapperService(createTempDir(), - Settings.builder() - .put("index.mapping.attachment.ignore_errors", false) - .build()).documentMapperParser(); + Settings.builder().put("index.mapping.attachment.ignore_errors", false).build(), + getIndicesModuleWithRegisteredAttachmentMapper()).documentMapperParser(); String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/attachment/test/unit/encrypted/test-mapping.json"); DocumentMapper docMapper = mapperParser.parse(mapping); diff --git a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/LanguageDetectionAttachmentMapperTests.java b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/LanguageDetectionAttachmentMapperTests.java index b2d361fe847..868ecb3ae55 100644 --- a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/LanguageDetectionAttachmentMapperTests.java +++ b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/LanguageDetectionAttachmentMapperTests.java @@ -21,11 +21,11 @@ package org.elasticsearch.mapper.attachments; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.index.MapperTestUtils; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.DocumentMapperParser; import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.mapper.core.StringFieldMapper; -import org.elasticsearch.mapper.attachments.AttachmentMapper; import org.junit.Before; import java.io.IOException; @@ -50,9 +50,8 @@ public class LanguageDetectionAttachmentMapperTests extends AttachmentUnitTestCa public void setupMapperParser(boolean langDetect) throws IOException { DocumentMapperParser mapperParser = MapperTestUtils.newMapperService(createTempDir(), - Settings.settingsBuilder() - .put("index.mapping.attachment.detect_language", langDetect) - .build()).documentMapperParser(); + Settings.settingsBuilder().put("index.mapping.attachment.detect_language", langDetect).build(), + getIndicesModuleWithRegisteredAttachmentMapper()).documentMapperParser(); String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/attachment/test/unit/language/language-mapping.json"); docMapper = mapperParser.parse(mapping); diff --git a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/MetadataMapperTests.java b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/MetadataMapperTests.java index cf2a130829f..acf0163acd9 100644 --- a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/MetadataMapperTests.java +++ b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/MetadataMapperTests.java @@ -21,11 +21,11 @@ package org.elasticsearch.mapper.attachments; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.MapperTestUtils; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.DocumentMapperParser; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.ParseContext; -import org.elasticsearch.mapper.attachments.AttachmentMapper; import java.io.IOException; @@ -44,7 +44,7 @@ public class MetadataMapperTests extends AttachmentUnitTestCase { .put(this.testSettings) .put(otherSettings) .build(); - DocumentMapperParser mapperParser = MapperTestUtils.newMapperService(createTempDir(), settings).documentMapperParser(); + DocumentMapperParser mapperParser = MapperTestUtils.newMapperService(createTempDir(), settings, getIndicesModuleWithRegisteredAttachmentMapper()).documentMapperParser(); String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/attachment/test/unit/metadata/test-mapping.json"); DocumentMapper docMapper = mapperParser.parse(mapping); diff --git a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/MultifieldAttachmentMapperTests.java b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/MultifieldAttachmentMapperTests.java index 4f070bd0dd1..40593ddb1bb 100644 --- a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/MultifieldAttachmentMapperTests.java +++ b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/MultifieldAttachmentMapperTests.java @@ -22,13 +22,13 @@ package org.elasticsearch.mapper.attachments; import org.elasticsearch.common.Base64; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.index.MapperTestUtils; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.DocumentMapperParser; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.index.mapper.core.DateFieldMapper; import org.elasticsearch.index.mapper.core.StringFieldMapper; -import org.elasticsearch.mapper.attachments.AttachmentMapper; import org.elasticsearch.threadpool.ThreadPool; import org.junit.After; import org.junit.Before; @@ -48,7 +48,7 @@ public class MultifieldAttachmentMapperTests extends AttachmentUnitTestCase { @Before public void setupMapperParser() throws Exception { - mapperParser = MapperTestUtils.newMapperService(createTempDir(), Settings.EMPTY).documentMapperParser(); + mapperParser = MapperTestUtils.newMapperService(createTempDir(), Settings.EMPTY, getIndicesModuleWithRegisteredAttachmentMapper()).documentMapperParser(); } @@ -91,7 +91,7 @@ public class MultifieldAttachmentMapperTests extends AttachmentUnitTestCase { String bytes = Base64.encodeBytes(originalText.getBytes(StandardCharsets.ISO_8859_1)); threadPool = new ThreadPool("testing-only"); - MapperService mapperService = MapperTestUtils.newMapperService(createTempDir(), Settings.EMPTY); + MapperService mapperService = MapperTestUtils.newMapperService(createTempDir(), Settings.EMPTY, getIndicesModuleWithRegisteredAttachmentMapper()); String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/attachment/test/unit/multifield/multifield-mapping.json"); diff --git a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/SimpleAttachmentMapperTests.java b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/SimpleAttachmentMapperTests.java index 0023fc44e24..01e87dc1430 100644 --- a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/SimpleAttachmentMapperTests.java +++ b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/SimpleAttachmentMapperTests.java @@ -25,6 +25,7 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.index.MapperTestUtils; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.DocumentMapperParser; import org.elasticsearch.index.mapper.MapperService; @@ -42,7 +43,7 @@ import static org.hamcrest.Matchers.*; public class SimpleAttachmentMapperTests extends AttachmentUnitTestCase { public void testSimpleMappings() throws Exception { - DocumentMapperParser mapperParser = MapperTestUtils.newMapperService(createTempDir(), Settings.EMPTY).documentMapperParser(); + DocumentMapperParser mapperParser = MapperTestUtils.newMapperService(createTempDir(), Settings.EMPTY, getIndicesModuleWithRegisteredAttachmentMapper()).documentMapperParser(); String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/attachment/test/unit/simple/test-mapping.json"); DocumentMapper docMapper = mapperParser.parse(mapping); byte[] html = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/attachment/test/sample-files/testXHTML.html"); @@ -69,9 +70,8 @@ public class SimpleAttachmentMapperTests extends AttachmentUnitTestCase { public void testContentBackcompat() throws Exception { DocumentMapperParser mapperParser = MapperTestUtils.newMapperService(createTempDir(), - Settings.builder() - .put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_1_4_2.id) - .build()).documentMapperParser(); + Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_1_4_2.id).build(), + getIndicesModuleWithRegisteredAttachmentMapper()).documentMapperParser(); String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/attachment/test/unit/simple/test-mapping.json"); DocumentMapper docMapper = mapperParser.parse(mapping); byte[] html = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/attachment/test/sample-files/testXHTML.html"); @@ -86,7 +86,7 @@ public class SimpleAttachmentMapperTests extends AttachmentUnitTestCase { * test for https://github.com/elastic/elasticsearch-mapper-attachments/issues/179 */ public void testSimpleMappingsWithAllFields() throws Exception { - DocumentMapperParser mapperParser = MapperTestUtils.newMapperService(createTempDir(), Settings.EMPTY).documentMapperParser(); + DocumentMapperParser mapperParser = MapperTestUtils.newMapperService(createTempDir(), Settings.EMPTY, getIndicesModuleWithRegisteredAttachmentMapper()).documentMapperParser(); String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/attachment/test/unit/simple/test-mapping-all-fields.json"); DocumentMapper docMapper = mapperParser.parse(mapping); byte[] html = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/attachment/test/sample-files/testXHTML.html"); @@ -131,7 +131,7 @@ public class SimpleAttachmentMapperTests extends AttachmentUnitTestCase { .endObject(); byte[] mapping = mappingBuilder.bytes().toBytes(); - MapperService mapperService = MapperTestUtils.newMapperService(createTempDir(), Settings.EMPTY); + MapperService mapperService = MapperTestUtils.newMapperService(createTempDir(), Settings.EMPTY, getIndicesModuleWithRegisteredAttachmentMapper()); DocumentMapper docMapper = mapperService.parse("mail", new CompressedXContent(mapping), true); // this should not throw an exception mapperService.parse("mail", new CompressedXContent(docMapper.mapping().toString()), true); diff --git a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/StandaloneRunner.java b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/StandaloneRunner.java index f6264337613..fcd430d0fbc 100644 --- a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/StandaloneRunner.java +++ b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/StandaloneRunner.java @@ -30,10 +30,10 @@ import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.env.Environment; +import org.elasticsearch.index.MapperTestUtils; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.DocumentMapperParser; import org.elasticsearch.index.mapper.ParseContext; -import org.elasticsearch.mapper.attachments.AttachmentMapper; import java.io.FileNotFoundException; import java.io.IOException; @@ -46,6 +46,7 @@ import static org.elasticsearch.common.cli.CliToolConfig.Builder.cmd; import static org.elasticsearch.common.cli.CliToolConfig.Builder.option; import static org.elasticsearch.common.io.Streams.copy; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.mapper.attachments.AttachmentUnitTestCase.getIndicesModuleWithRegisteredAttachmentMapper; import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath; /** @@ -88,7 +89,7 @@ public class StandaloneRunner extends CliTool { this.size = size; this.url = url; this.base64text = base64text; - DocumentMapperParser mapperParser = MapperTestUtils.newMapperService(PathUtils.get("."), Settings.EMPTY).documentMapperParser(); // use CWD b/c it won't be used + DocumentMapperParser mapperParser = MapperTestUtils.newMapperService(PathUtils.get("."), Settings.EMPTY, getIndicesModuleWithRegisteredAttachmentMapper()).documentMapperParser(); // use CWD b/c it won't be used String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/attachment/test/standalone/standalone-mapping.json"); docMapper = mapperParser.parse(mapping); diff --git a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/VariousDocTests.java b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/VariousDocTests.java index c2569fd3663..5341e038cff 100644 --- a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/VariousDocTests.java +++ b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/VariousDocTests.java @@ -23,10 +23,10 @@ import org.apache.tika.io.IOUtils; import org.apache.tika.metadata.Metadata; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.MapperTestUtils; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.DocumentMapperParser; import org.elasticsearch.index.mapper.ParseContext; -import org.elasticsearch.mapper.attachments.AttachmentMapper; import org.junit.Before; import java.io.IOException; @@ -48,7 +48,7 @@ public class VariousDocTests extends AttachmentUnitTestCase { @Before public void createMapper() throws IOException { - DocumentMapperParser mapperParser = MapperTestUtils.newMapperService(createTempDir(), Settings.EMPTY).documentMapperParser(); + DocumentMapperParser mapperParser = MapperTestUtils.newMapperService(createTempDir(), Settings.EMPTY, getIndicesModuleWithRegisteredAttachmentMapper()).documentMapperParser(); String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/attachment/test/unit/various-doc/test-mapping.json"); docMapper = mapperParser.parse(mapping); @@ -93,7 +93,7 @@ public class VariousDocTests extends AttachmentUnitTestCase { assertParseable("text-in-english.txt"); testMapper("text-in-english.txt", false); } - + /** * Test for .epub */ @@ -129,7 +129,7 @@ public class VariousDocTests extends AttachmentUnitTestCase { protected void assertParseable(String filename) throws Exception { try (InputStream is = VariousDocTests.class.getResourceAsStream("/org/elasticsearch/index/mapper/attachment/test/sample-files/" + filename)) { byte bytes[] = IOUtils.toByteArray(is); - String parsedContent = TikaImpl.parse(bytes, new Metadata(), -1); + String parsedContent = TikaImpl.parse(bytes, new Metadata(), -1); assertThat(parsedContent, not(isEmptyOrNullString())); logger.debug("extracted content: {}", parsedContent); } diff --git a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/MapperTestUtils.java b/test-framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java similarity index 66% rename from plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/MapperTestUtils.java rename to test-framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java index 1513f7ba6e0..8b529f9fc8f 100644 --- a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/MapperTestUtils.java +++ b/test-framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java @@ -17,14 +17,12 @@ * under the License. */ -package org.elasticsearch.mapper.attachments; +package org.elasticsearch.index; import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.AnalysisRegistry; import org.elasticsearch.index.analysis.AnalysisService; import org.elasticsearch.index.mapper.MapperService; @@ -37,22 +35,29 @@ import java.io.IOException; import java.nio.file.Path; import java.util.Collections; -class MapperTestUtils { + +public class MapperTestUtils { public static MapperService newMapperService(Path tempDir, Settings indexSettings) throws IOException { - Settings nodeSettings = Settings.builder() - .put("path.home", tempDir) - .build(); - indexSettings = Settings.builder() - .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) - .put(indexSettings) - .build(); - IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(new Index("test"), indexSettings); - AnalysisService analysisService = new AnalysisRegistry(null, new Environment(nodeSettings)).build(idxSettings); - SimilarityService similarityService = new SimilarityService(idxSettings, Collections.emptyMap()); IndicesModule indicesModule = new IndicesModule(); - indicesModule.registerMapper(AttachmentMapper.CONTENT_TYPE, new AttachmentMapper.TypeParser()); + return newMapperService(tempDir, indexSettings, indicesModule); + } + + public static MapperService newMapperService(Path tempDir, Settings settings, IndicesModule indicesModule) throws IOException { + Settings.Builder settingsBuilder = Settings.builder() + .put("path.home", tempDir) + .put(settings); + if (settings.get(IndexMetaData.SETTING_VERSION_CREATED) == null) { + settingsBuilder.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT); + } + Settings finalSettings = settingsBuilder.build(); MapperRegistry mapperRegistry = indicesModule.getMapperRegistry(); - return new MapperService(idxSettings, analysisService, similarityService, mapperRegistry); + IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(new Index("test"), finalSettings); + AnalysisService analysisService = new AnalysisRegistry(null, new Environment(finalSettings)).build(indexSettings); + SimilarityService similarityService = new SimilarityService(indexSettings, Collections.emptyMap()); + return new MapperService(indexSettings, + analysisService, + similarityService, + mapperRegistry); } }