diff --git a/.gitignore b/.gitignore index 6f32f60a650..dbd3a1442cf 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,6 @@ /.local-execution-hints.log /.local-*-execution-hints.log /eclipse-build/ +build/ +generated-resources/ +.gradle/ diff --git a/build.gradle b/build.gradle new file mode 100644 index 00000000000..ac98659d527 --- /dev/null +++ b/build.gradle @@ -0,0 +1,83 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.elasticsearch.gradle.ElasticsearchProperties + +buildscript { + repositories { + mavenCentral() + maven { + name 'sonatype-snapshots' + url 'http://oss.sonatype.org/content/repositories/snapshots/' + } + jcenter() + } + dependencies { + classpath 'org.elasticsearch.gradle:build-tools:3.0.0-SNAPSHOT' + } +} +apply plugin: 'idea' +apply plugin: 'eclipse' +apply plugin: 'elasticsearch.esplugin' + +esplugin { + description 'The mapper attachments plugin adds the attachment type to Elasticsearch using Apache Tika.' + classname 'org.elasticsearch.plugin.mapper.attachments.MapperAttachmentsPlugin' +} + +project.group = 'org.elasticsearch' +project.version = ElasticsearchProperties.version +project.ext.luceneVersion = ElasticsearchProperties.luceneVersion +repositories { + mavenCentral() + maven { + name 'sonatype-snapshots' + url 'http://oss.sonatype.org/content/repositories/snapshots/' + } + if (luceneVersion.contains('-snapshot')) { + String revision = (luceneVersion =~ /\d\.\d\.\d-snapshot-(\d+)/)[0][1] + maven { + name 'lucene-snapshots' + url "http://s3.amazonaws.com/download.elasticsearch.org/lucenesnapshots/${revision}" + } + } +} + +dependencies { + compile('org.apache.tika:tika-parsers:1.10') { + // TODO: is all of edu.ucar incompatibile with apache2 license? if so, we can exclude the group? + // Not Apache2 License compatible + exclude group: 'edu.ucar', module: 'netcdf' + exclude group: 'edu.ucar', module: 'cdm' + exclude group: 'edu.ucar', module: 'httpservices' + exclude group: 'edu.ucar', module: 'grib' + exclude group: 'edu.ucar', module: 'netcdf4' + exclude group: 'com.uwyn', module: 'jhighlight' + // ES core already has these + exclude group: 'org.aw2.asm', module: 'asm-debug-all' + exclude group: 'commons-logging', module: 'commons-logging-api' + } +} + +compileJava.options.compilerArgs << '-Xlint:-cast,-deprecation,-rawtypes' + +forbiddenPatterns { + exclude '**/*.docx' + exclude '**/*.pdf' +} diff --git a/pom.xml b/pom.xml deleted file mode 100644 index 2d5761db27c..00000000000 --- a/pom.xml +++ /dev/null @@ -1,118 +0,0 @@ - - - 4.0.0 - - org.elasticsearch - elasticsearch-mapper-attachments - 3.1.0-SNAPSHOT - Elasticsearch Mapper Attachment plugin - The mapper attachments plugin adds the attachment type to Elasticsearch using Apache Tika. - https://github.com/elastic/elasticsearch-mapper-attachments/ - 2009 - - - The Apache Software License, Version 2.0 - http://www.apache.org/licenses/LICENSE-2.0.txt - repo - - - - scm:git:git@github.com:elastic/elasticsearch-mapper-attachments.git - scm:git:git@github.com:elastic/elasticsearch-mapper-attachments.git - http://github.com/elastic/elasticsearch-mapper-attachments - - - - org.elasticsearch.plugin - plugins - 2.1.0-SNAPSHOT - - - - 2.1.0-SNAPSHOT - mapper-attachments - org.elasticsearch.plugin.mapper.attachments.MapperAttachmentsPlugin - ${project.basedir}/src/main/assemblies/plugin.xml - mapper_attachments - false - - ${project.basedir}/src/test/resources/integ-tests-183.xml - - - - - org.elasticsearch - elasticsearch - ${elasticsearch.version} - provided - - - - - org.apache.tika - tika-parsers - 1.10 - - - - edu.ucar - netcdf - - - - edu.ucar - cdm - - - - edu.ucar - httpservices - - - - edu.ucar - grib - - - - edu.ucar - netcdf4 - - - - com.uwyn - jhighlight - - - - org.ow2.asm - asm-debug-all - - - commons-logging - commons-logging-api - - - - - - - - - org.apache.maven.plugins - maven-assembly-plugin - - - - - - - - oss-snapshots - Sonatype OSS Snapshots - https://oss.sonatype.org/content/repositories/snapshots/ - - - diff --git a/src/main/java/org/elasticsearch/index/mapper/attachment/AttachmentMapper.java b/src/main/java/org/elasticsearch/index/mapper/attachment/AttachmentMapper.java index 69dcca8609f..e4250c14d9b 100644 --- a/src/main/java/org/elasticsearch/index/mapper/attachment/AttachmentMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/attachment/AttachmentMapper.java @@ -26,6 +26,7 @@ import org.apache.tika.Tika; import org.apache.tika.language.LanguageIdentifier; import org.apache.tika.metadata.Metadata; import org.elasticsearch.Version; +import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.logging.ESLoggerFactory; @@ -58,7 +59,7 @@ import static org.elasticsearch.plugin.mapper.attachments.tika.TikaInstance.tika * } * } * - *

+ *

* _content_length = Specify the maximum amount of characters to extract from the attachment. If not specified, then the default for * tika is 100,000 characters. Caution is required when setting large values as this can cause memory issues. */ @@ -624,8 +625,9 @@ public class AttachmentMapper extends FieldMapper { } @Override + @SuppressWarnings("unchecked") public Iterator iterator() { - List extras = Arrays.asList( + List extras = Arrays.asList( contentMapper, dateMapper, titleMapper, @@ -635,7 +637,7 @@ public class AttachmentMapper extends FieldMapper { contentTypeMapper, contentLengthMapper, languageMapper); - return CollectionUtils.concat(super.iterator(), extras.iterator()); + return Iterators.concat(super.iterator(), extras.iterator()); } @Override diff --git a/src/main/java/org/elasticsearch/index/mapper/attachment/RegisterAttachmentType.java b/src/main/java/org/elasticsearch/index/mapper/attachment/RegisterAttachmentType.java deleted file mode 100644 index 85b489bbd4c..00000000000 --- a/src/main/java/org/elasticsearch/index/mapper/attachment/RegisterAttachmentType.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.mapper.attachment; - -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.AbstractIndexComponent; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.settings.IndexSettings; - -/** - * - */ -public class RegisterAttachmentType extends AbstractIndexComponent { - - @Inject - public RegisterAttachmentType(Index index, @IndexSettings Settings indexSettings, MapperService mapperService) { - super(index, indexSettings); - - mapperService.documentMapperParser().putTypeParser("attachment", new AttachmentMapper.TypeParser()); - } -} diff --git a/src/main/java/org/elasticsearch/plugin/mapper/attachments/AttachmentsIndexModule.java b/src/main/java/org/elasticsearch/plugin/mapper/attachments/AttachmentsIndexModule.java deleted file mode 100644 index 678857d89fe..00000000000 --- a/src/main/java/org/elasticsearch/plugin/mapper/attachments/AttachmentsIndexModule.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.plugin.mapper.attachments; - -import org.elasticsearch.common.inject.AbstractModule; -import org.elasticsearch.index.mapper.attachment.RegisterAttachmentType; - -/** - * - */ -public class AttachmentsIndexModule extends AbstractModule { - - @Override - protected void configure() { - bind(RegisterAttachmentType.class).asEagerSingleton(); - } -} diff --git a/src/main/java/org/elasticsearch/plugin/mapper/attachments/MapperAttachmentsPlugin.java b/src/main/java/org/elasticsearch/plugin/mapper/attachments/MapperAttachmentsPlugin.java index f5673d7815a..c2d20d7429b 100644 --- a/src/main/java/org/elasticsearch/plugin/mapper/attachments/MapperAttachmentsPlugin.java +++ b/src/main/java/org/elasticsearch/plugin/mapper/attachments/MapperAttachmentsPlugin.java @@ -19,16 +19,10 @@ package org.elasticsearch.plugin.mapper.attachments; -import org.elasticsearch.common.inject.Module; -import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexService; +import org.elasticsearch.index.mapper.attachment.AttachmentMapper; import org.elasticsearch.plugins.Plugin; -import java.util.Collection; -import java.util.Collections; - -/** - * - */ public class MapperAttachmentsPlugin extends Plugin { @Override @@ -42,7 +36,7 @@ public class MapperAttachmentsPlugin extends Plugin { } @Override - public Collection indexModules(Settings indexSettings) { - return Collections.singletonList(new AttachmentsIndexModule()); + public void onIndexService(IndexService indexService) { + indexService.mapperService().documentMapperParser().putTypeParser("attachment", new AttachmentMapper.TypeParser()); } } diff --git a/src/test/java/org/elasticsearch/index/mapper/attachment/test/MapperTestUtils.java b/src/test/java/org/elasticsearch/index/mapper/attachment/test/MapperTestUtils.java index 7f73d4313d3..8aff7f04748 100644 --- a/src/test/java/org/elasticsearch/index/mapper/attachment/test/MapperTestUtils.java +++ b/src/test/java/org/elasticsearch/index/mapper/attachment/test/MapperTestUtils.java @@ -22,23 +22,19 @@ package org.elasticsearch.index.mapper.attachment.test; import org.apache.lucene.util.Constants; import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.common.inject.Injector; -import org.elasticsearch.common.inject.ModulesBuilder; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.settings.SettingsModule; import org.elasticsearch.env.Environment; -import org.elasticsearch.env.EnvironmentModule; import org.elasticsearch.index.Index; -import org.elasticsearch.index.IndexNameModule; -import org.elasticsearch.index.analysis.AnalysisModule; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.analysis.AnalysisRegistry; import org.elasticsearch.index.analysis.AnalysisService; -import org.elasticsearch.index.mapper.DocumentMapperParser; import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.settings.IndexSettingsModule; -import org.elasticsearch.index.similarity.SimilarityLookupService; -import org.elasticsearch.indices.analysis.IndicesAnalysisService; +import org.elasticsearch.index.similarity.SimilarityService; +import org.elasticsearch.test.IndexSettingsModule; +import java.io.IOException; import java.nio.file.Path; +import java.util.Collections; import java.util.Locale; import static com.carrotsearch.randomizedtesting.RandomizedTest.assumeTrue; @@ -46,56 +42,18 @@ import static org.elasticsearch.plugin.mapper.attachments.tika.LocaleChecker.isL public class MapperTestUtils { - public static MapperService newMapperService(Path tempDir) { - return newMapperService(new Index("test"), Settings.builder() - .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) - .put("path.home", tempDir) - .build()); - } - - public static MapperService newMapperService(Index index, Settings indexSettings) { - return new MapperService(index, - indexSettings, - newAnalysisService(indexSettings), - newSimilarityLookupService(indexSettings), - null); - } - - public static AnalysisService newAnalysisService(Path tempDir) { - return newAnalysisService(Settings.builder() - .put("path.home", tempDir) - .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) - .build()); - } - - public static AnalysisService newAnalysisService(Settings indexSettings) { - Injector parentInjector = new ModulesBuilder().add(new SettingsModule(indexSettings), new EnvironmentModule(new Environment(indexSettings))).createInjector(); - Index index = new Index("test"); - Injector injector = new ModulesBuilder().add( - new IndexSettingsModule(index, indexSettings), - new IndexNameModule(index), - new AnalysisModule(indexSettings, parentInjector.getInstance(IndicesAnalysisService.class))).createChildInjector(parentInjector); - - return injector.getInstance(AnalysisService.class); - } - - public static SimilarityLookupService newSimilarityLookupService(Settings indexSettings) { - return new SimilarityLookupService(new Index("test"), indexSettings); - } - - public static DocumentMapperParser newMapperParser(Path tempDir) { - return newMapperParser(Settings.builder() - .put("path.home", tempDir) - .build()); - } - - public static DocumentMapperParser newMapperParser(Settings settings) { - Settings forcedSettings = Settings.builder() - .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) - .put(settings) - .build(); - MapperService mapperService = new MapperService(new Index("test"), forcedSettings, newAnalysisService(forcedSettings), newSimilarityLookupService(forcedSettings), null); - return new DocumentMapperParser(forcedSettings, mapperService, MapperTestUtils.newAnalysisService(forcedSettings), null, null); + public static MapperService newMapperService(Path tempDir, Settings indexSettings) throws IOException { + Settings nodeSettings = Settings.builder() + .put("path.home", tempDir) + .build(); + indexSettings = Settings.builder() + .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) + .put(indexSettings) + .build(); + IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(new Index("test"), indexSettings, Collections.emptyList()); + AnalysisService analysisService = new AnalysisRegistry(null, new Environment(nodeSettings)).build(idxSettings); + SimilarityService similarityService = new SimilarityService(idxSettings, Collections.emptyMap()); + return new MapperService(idxSettings, analysisService, similarityService); } /** diff --git a/src/test/java/org/elasticsearch/index/mapper/attachment/test/standalone/StandaloneRunner.java b/src/test/java/org/elasticsearch/index/mapper/attachment/test/standalone/StandaloneRunner.java index b16f98e3b58..11382a9a536 100644 --- a/src/test/java/org/elasticsearch/index/mapper/attachment/test/standalone/StandaloneRunner.java +++ b/src/test/java/org/elasticsearch/index/mapper/attachment/test/standalone/StandaloneRunner.java @@ -87,7 +87,7 @@ public class StandaloneRunner extends CliTool { this.size = size; this.url = url; this.base64text = base64text; - DocumentMapperParser mapperParser = MapperTestUtils.newMapperParser(PathUtils.get(".")); // use CWD b/c it won't be used + DocumentMapperParser mapperParser = MapperTestUtils.newMapperService(PathUtils.get("."), Settings.EMPTY).documentMapperParser(); // use CWD b/c it won't be used mapperParser.putTypeParser(AttachmentMapper.CONTENT_TYPE, new AttachmentMapper.TypeParser()); String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/attachment/test/standalone/standalone-mapping.json"); diff --git a/src/test/java/org/elasticsearch/index/mapper/attachment/test/unit/DateAttachmentMapperTests.java b/src/test/java/org/elasticsearch/index/mapper/attachment/test/unit/DateAttachmentMapperTests.java index caead2bb585..727c7f24847 100644 --- a/src/test/java/org/elasticsearch/index/mapper/attachment/test/unit/DateAttachmentMapperTests.java +++ b/src/test/java/org/elasticsearch/index/mapper/attachment/test/unit/DateAttachmentMapperTests.java @@ -19,6 +19,7 @@ package org.elasticsearch.index.mapper.attachment.test.unit; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.DocumentMapperParser; import org.elasticsearch.index.mapper.attachment.AttachmentMapper; @@ -38,12 +39,11 @@ public class DateAttachmentMapperTests extends AttachmentUnitTestCase { private DocumentMapperParser mapperParser; @Before - public void setupMapperParser() { - mapperParser = MapperTestUtils.newMapperParser(createTempDir()); + public void setupMapperParser() throws Exception { + mapperParser = MapperTestUtils.newMapperService(createTempDir(), Settings.EMPTY).documentMapperParser(); mapperParser.putTypeParser(AttachmentMapper.CONTENT_TYPE, new AttachmentMapper.TypeParser()); } - @Test public void testSimpleMappings() throws Exception { String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/attachment/test/unit/date/date-mapping.json"); DocumentMapper docMapper = mapperParser.parse(mapping); diff --git a/src/test/java/org/elasticsearch/index/mapper/attachment/test/unit/EncryptedDocMapperTest.java b/src/test/java/org/elasticsearch/index/mapper/attachment/test/unit/EncryptedDocMapperTest.java index eba07b23859..af53482a571 100644 --- a/src/test/java/org/elasticsearch/index/mapper/attachment/test/unit/EncryptedDocMapperTest.java +++ b/src/test/java/org/elasticsearch/index/mapper/attachment/test/unit/EncryptedDocMapperTest.java @@ -43,9 +43,8 @@ import static org.hamcrest.Matchers.*; */ public class EncryptedDocMapperTest extends AttachmentUnitTestCase { - @Test public void testMultipleDocsEncryptedLast() throws IOException { - DocumentMapperParser mapperParser = MapperTestUtils.newMapperParser(createTempDir()); + DocumentMapperParser mapperParser = MapperTestUtils.newMapperService(createTempDir(), Settings.EMPTY).documentMapperParser(); mapperParser.putTypeParser(AttachmentMapper.CONTENT_TYPE, new AttachmentMapper.TypeParser()); String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/attachment/test/unit/encrypted/test-mapping.json"); @@ -75,9 +74,8 @@ public class EncryptedDocMapperTest extends AttachmentUnitTestCase { assertThat(doc.getField(docMapper.mappers().getMapper("file2.content_length").fieldType().names().indexName()), nullValue()); } - @Test public void testMultipleDocsEncryptedFirst() throws IOException { - DocumentMapperParser mapperParser = MapperTestUtils.newMapperParser(createTempDir()); + DocumentMapperParser mapperParser = MapperTestUtils.newMapperService(createTempDir(), Settings.EMPTY).documentMapperParser(); mapperParser.putTypeParser(AttachmentMapper.CONTENT_TYPE, new AttachmentMapper.TypeParser()); String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/attachment/test/unit/encrypted/test-mapping.json"); @@ -107,40 +105,31 @@ public class EncryptedDocMapperTest extends AttachmentUnitTestCase { assertThat(doc.getField(docMapper.mappers().getMapper("file2.content_length").fieldType().names().indexName()).numericValue().longValue(), is(344L)); } - @Test(expected = MapperParsingException.class) public void testMultipleDocsEncryptedNotIgnoringErrors() throws IOException { - DocumentMapperParser mapperParser = MapperTestUtils.newMapperParser( - Settings.builder() - .put("path.home", createTempDir()) - .put("index.mapping.attachment.ignore_errors", false) - .build()); - mapperParser.putTypeParser(AttachmentMapper.CONTENT_TYPE, new AttachmentMapper.TypeParser()); + try { + DocumentMapperParser mapperParser = MapperTestUtils.newMapperService(createTempDir(), + Settings.builder() + .put("index.mapping.attachment.ignore_errors", false) + .build()).documentMapperParser(); + mapperParser.putTypeParser(AttachmentMapper.CONTENT_TYPE, new AttachmentMapper.TypeParser()); - String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/attachment/test/unit/encrypted/test-mapping.json"); - DocumentMapper docMapper = mapperParser.parse(mapping); - byte[] html = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/attachment/test/sample-files/htmlWithValidDateMeta.html"); - byte[] pdf = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/attachment/test/sample-files/encrypted.pdf"); + String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/attachment/test/unit/encrypted/test-mapping.json"); + DocumentMapper docMapper = mapperParser.parse(mapping); + byte[] html = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/attachment/test/sample-files/htmlWithValidDateMeta.html"); + byte[] pdf = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/attachment/test/sample-files/encrypted.pdf"); - BytesReference json = jsonBuilder() + BytesReference json = jsonBuilder() .startObject() .field("file1", pdf) .field("file2", html) .endObject().bytes(); - ParseContext.Document doc = docMapper.parse("person", "person", "1", json).rootDoc(); - assertThat(doc.get(docMapper.mappers().getMapper("file1").fieldType().names().indexName()), nullValue()); - assertThat(doc.get(docMapper.mappers().getMapper("file1.title").fieldType().names().indexName()), nullValue()); - assertThat(doc.get(docMapper.mappers().getMapper("file1.author").fieldType().names().indexName()), nullValue()); - assertThat(doc.get(docMapper.mappers().getMapper("file1.keywords").fieldType().names().indexName()), nullValue()); - assertThat(doc.get(docMapper.mappers().getMapper("file1.content_type").fieldType().names().indexName()), nullValue()); - assertThat(doc.getField(docMapper.mappers().getMapper("file1.content_length").fieldType().names().indexName()), nullValue()); - - assertThat(doc.get(docMapper.mappers().getMapper("file2").fieldType().names().indexName()), containsString("World")); - assertThat(doc.get(docMapper.mappers().getMapper("file2.title").fieldType().names().indexName()), equalTo("Hello")); - assertThat(doc.get(docMapper.mappers().getMapper("file2.author").fieldType().names().indexName()), equalTo("kimchy")); - assertThat(doc.get(docMapper.mappers().getMapper("file2.keywords").fieldType().names().indexName()), equalTo("elasticsearch,cool,bonsai")); - assertThat(doc.get(docMapper.mappers().getMapper("file2.content_type").fieldType().names().indexName()), equalTo("text/html; charset=ISO-8859-1")); - assertThat(doc.getField(docMapper.mappers().getMapper("file2.content_length").fieldType().names().indexName()).numericValue().longValue(), is(344L)); + docMapper.parse("person", "person", "1", json); + fail("Expected doc parsing exception"); + } catch (MapperParsingException e) { + // TODO: check the error message...getting security problems atm + //assertTrue(e.getMessage(), e.getMessage().contains()) + } } } diff --git a/src/test/java/org/elasticsearch/index/mapper/attachment/test/unit/LanguageDetectionAttachmentMapperTests.java b/src/test/java/org/elasticsearch/index/mapper/attachment/test/unit/LanguageDetectionAttachmentMapperTests.java index df76dbdb193..5664802daa4 100644 --- a/src/test/java/org/elasticsearch/index/mapper/attachment/test/unit/LanguageDetectionAttachmentMapperTests.java +++ b/src/test/java/org/elasticsearch/index/mapper/attachment/test/unit/LanguageDetectionAttachmentMapperTests.java @@ -51,11 +51,10 @@ public class LanguageDetectionAttachmentMapperTests extends AttachmentUnitTestCa } public void setupMapperParser(boolean langDetect) throws IOException { - DocumentMapperParser mapperParser = MapperTestUtils.newMapperParser( - Settings.settingsBuilder() - .put("path.home", createTempDir()) - .put("index.mapping.attachment.detect_language", langDetect) - .build()); + DocumentMapperParser mapperParser = MapperTestUtils.newMapperService(createTempDir(), + Settings.settingsBuilder() + .put("index.mapping.attachment.detect_language", langDetect) + .build()).documentMapperParser(); mapperParser.putTypeParser(AttachmentMapper.CONTENT_TYPE, new AttachmentMapper.TypeParser()); String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/attachment/test/unit/language/language-mapping.json"); docMapper = mapperParser.parse(mapping); @@ -84,38 +83,31 @@ public class LanguageDetectionAttachmentMapperTests extends AttachmentUnitTestCa assertThat(doc.get(docMapper.mappers().getMapper("file.language").fieldType().names().indexName()), equalTo(expected)); } - @Test public void testFrDetection() throws Exception { testLanguage("text-in-french.txt", "fr"); } - @Test public void testEnDetection() throws Exception { testLanguage("text-in-english.txt", "en"); } - @Test public void testFrForced() throws Exception { testLanguage("text-in-english.txt", "fr", "fr"); } /** * This test gives strange results! detection of ":-)" gives "lt" as a result - * @throws Exception */ - @Test public void testNoLanguage() throws Exception { testLanguage("text-in-nolang.txt", "lt"); } - @Test public void testLangDetectDisabled() throws Exception { // We replace the mapper with another one which have index.mapping.attachment.detect_language = false setupMapperParser(false); testLanguage("text-in-english.txt", null); } - @Test public void testLangDetectDocumentEnabled() throws Exception { // We replace the mapper with another one which have index.mapping.attachment.detect_language = false setupMapperParser(false); diff --git a/src/test/java/org/elasticsearch/index/mapper/attachment/test/unit/MetadataMapperTest.java b/src/test/java/org/elasticsearch/index/mapper/attachment/test/unit/MetadataMapperTest.java index 0067bf96caa..4c541dd6e0d 100644 --- a/src/test/java/org/elasticsearch/index/mapper/attachment/test/unit/MetadataMapperTest.java +++ b/src/test/java/org/elasticsearch/index/mapper/attachment/test/unit/MetadataMapperTest.java @@ -46,7 +46,7 @@ public class MetadataMapperTest extends AttachmentUnitTestCase { .put(this.testSettings) .put(otherSettings) .build(); - DocumentMapperParser mapperParser = MapperTestUtils.newMapperParser(settings); + DocumentMapperParser mapperParser = MapperTestUtils.newMapperService(createTempDir(), settings).documentMapperParser(); mapperParser.putTypeParser(AttachmentMapper.CONTENT_TYPE, new AttachmentMapper.TypeParser()); String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/attachment/test/unit/metadata/test-mapping.json"); @@ -76,32 +76,30 @@ public class MetadataMapperTest extends AttachmentUnitTestCase { assertThat(doc.getField(docMapper.mappers().getMapper("file.content_length").fieldType().names().indexName()).numericValue().longValue(), is(expectedLength)); } - @Test public void testIgnoreWithoutDate() throws Exception { checkMeta("htmlWithoutDateMeta.html", Settings.builder().build(), null, 300L); } - @Test public void testIgnoreWithEmptyDate() throws Exception { checkMeta("htmlWithEmptyDateMeta.html", Settings.builder().build(), null, 334L); } - @Test public void testIgnoreWithCorrectDate() throws Exception { checkMeta("htmlWithValidDateMeta.html", Settings.builder().build(), 1354233600000L, 344L); } - @Test public void testWithoutDate() throws Exception { checkMeta("htmlWithoutDateMeta.html", Settings.builder().put("index.mapping.attachment.ignore_errors", false).build(), null, 300L); } - @Test(expected = MapperParsingException.class) public void testWithEmptyDate() throws Exception { - checkMeta("htmlWithEmptyDateMeta.html", Settings.builder().put("index.mapping.attachment.ignore_errors", false).build(), null, null); + try { + checkMeta("htmlWithEmptyDateMeta.html", Settings.builder().put("index.mapping.attachment.ignore_errors", false).build(), null, null); + } catch (MapperParsingException e) { + throw e; + } } - @Test public void testWithCorrectDate() throws Exception { checkMeta("htmlWithValidDateMeta.html", Settings.builder().put("index.mapping.attachment.ignore_errors", false).build(), 1354233600000L, 344L); } diff --git a/src/test/java/org/elasticsearch/index/mapper/attachment/test/unit/MultifieldAttachmentMapperTests.java b/src/test/java/org/elasticsearch/index/mapper/attachment/test/unit/MultifieldAttachmentMapperTests.java index 82e3d67083f..415a977793d 100644 --- a/src/test/java/org/elasticsearch/index/mapper/attachment/test/unit/MultifieldAttachmentMapperTests.java +++ b/src/test/java/org/elasticsearch/index/mapper/attachment/test/unit/MultifieldAttachmentMapperTests.java @@ -20,6 +20,7 @@ package org.elasticsearch.index.mapper.attachment.test.unit; import org.elasticsearch.common.Base64; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.DocumentMapperParser; @@ -48,8 +49,8 @@ public class MultifieldAttachmentMapperTests extends AttachmentUnitTestCase { private ThreadPool threadPool; @Before - public void setupMapperParser() { - mapperParser = MapperTestUtils.newMapperParser(createTempDir()); + public void setupMapperParser() throws Exception { + mapperParser = MapperTestUtils.newMapperService(createTempDir(), Settings.EMPTY).documentMapperParser(); mapperParser.putTypeParser(AttachmentMapper.CONTENT_TYPE, new AttachmentMapper.TypeParser()); } @@ -59,7 +60,6 @@ public class MultifieldAttachmentMapperTests extends AttachmentUnitTestCase { terminate(threadPool); } - @Test public void testSimpleMappings() throws Exception { String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/attachment/test/unit/multifield/multifield-mapping.json"); DocumentMapper docMapper = mapperParser.parse(mapping); @@ -87,7 +87,6 @@ public class MultifieldAttachmentMapperTests extends AttachmentUnitTestCase { assertThat(docMapper.mappers().getMapper("file.content_type.suggest"), instanceOf(StringFieldMapper.class)); } - @Test public void testExternalValues() throws Exception { String originalText = "This is an elasticsearch mapper attachment test."; String contentType = "text/plain; charset=ISO-8859-1"; @@ -96,7 +95,7 @@ public class MultifieldAttachmentMapperTests extends AttachmentUnitTestCase { String bytes = Base64.encodeBytes(originalText.getBytes(StandardCharsets.ISO_8859_1)); threadPool = new ThreadPool("testing-only"); - MapperService mapperService = MapperTestUtils.newMapperService(createTempDir()); + MapperService mapperService = MapperTestUtils.newMapperService(createTempDir(), Settings.EMPTY); mapperService.documentMapperParser().putTypeParser(AttachmentMapper.CONTENT_TYPE, new AttachmentMapper.TypeParser()); String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/attachment/test/unit/multifield/multifield-mapping.json"); diff --git a/src/test/java/org/elasticsearch/index/mapper/attachment/test/unit/SimpleAttachmentMapperTests.java b/src/test/java/org/elasticsearch/index/mapper/attachment/test/unit/SimpleAttachmentMapperTests.java index 8a5a925b038..20e91e9a584 100644 --- a/src/test/java/org/elasticsearch/index/mapper/attachment/test/unit/SimpleAttachmentMapperTests.java +++ b/src/test/java/org/elasticsearch/index/mapper/attachment/test/unit/SimpleAttachmentMapperTests.java @@ -40,7 +40,7 @@ import static org.hamcrest.Matchers.*; public class SimpleAttachmentMapperTests extends AttachmentUnitTestCase { public void testSimpleMappings() throws Exception { - DocumentMapperParser mapperParser = MapperTestUtils.newMapperParser(createTempDir()); + DocumentMapperParser mapperParser = MapperTestUtils.newMapperService(createTempDir(), Settings.EMPTY).documentMapperParser(); mapperParser.putTypeParser(AttachmentMapper.CONTENT_TYPE, new AttachmentMapper.TypeParser()); String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/attachment/test/unit/simple/test-mapping.json"); DocumentMapper docMapper = mapperParser.parse(mapping); @@ -67,10 +67,10 @@ public class SimpleAttachmentMapperTests extends AttachmentUnitTestCase { } public void testContentBackcompat() throws Exception { - DocumentMapperParser mapperParser = MapperTestUtils.newMapperParser(Settings.builder() - .put("path.home", createTempDir()) - .put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_1_4_2.id) - .build()); + DocumentMapperParser mapperParser = MapperTestUtils.newMapperService(createTempDir(), + Settings.builder() + .put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_1_4_2.id) + .build()).documentMapperParser(); mapperParser.putTypeParser(AttachmentMapper.CONTENT_TYPE, new AttachmentMapper.TypeParser()); String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/attachment/test/unit/simple/test-mapping.json"); DocumentMapper docMapper = mapperParser.parse(mapping); @@ -84,10 +84,9 @@ public class SimpleAttachmentMapperTests extends AttachmentUnitTestCase { /** * test for https://github.com/elastic/elasticsearch-mapper-attachments/issues/179 - * @throws Exception */ public void testSimpleMappingsWithAllFields() throws Exception { - DocumentMapperParser mapperParser = MapperTestUtils.newMapperParser(createTempDir()); + DocumentMapperParser mapperParser = MapperTestUtils.newMapperService(createTempDir(), Settings.EMPTY).documentMapperParser(); mapperParser.putTypeParser(AttachmentMapper.CONTENT_TYPE, new AttachmentMapper.TypeParser()); String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/attachment/test/unit/simple/test-mapping-all-fields.json"); DocumentMapper docMapper = mapperParser.parse(mapping); diff --git a/src/test/java/org/elasticsearch/index/mapper/attachment/test/unit/VariousDocTest.java b/src/test/java/org/elasticsearch/index/mapper/attachment/test/unit/VariousDocTest.java index e8e2e919bab..3d7fee6c627 100644 --- a/src/test/java/org/elasticsearch/index/mapper/attachment/test/unit/VariousDocTest.java +++ b/src/test/java/org/elasticsearch/index/mapper/attachment/test/unit/VariousDocTest.java @@ -21,6 +21,7 @@ package org.elasticsearch.index.mapper.attachment.test.unit; import org.apache.tika.Tika; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.DocumentMapperParser; import org.elasticsearch.index.mapper.ParseContext; @@ -49,7 +50,7 @@ public class VariousDocTest extends AttachmentUnitTestCase { @Before public void createMapper() throws IOException { - DocumentMapperParser mapperParser = MapperTestUtils.newMapperParser(createTempDir()); + DocumentMapperParser mapperParser = MapperTestUtils.newMapperService(createTempDir(), Settings.EMPTY).documentMapperParser(); mapperParser.putTypeParser(AttachmentMapper.CONTENT_TYPE, new AttachmentMapper.TypeParser()); String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/attachment/test/unit/various-doc/test-mapping.json"); @@ -59,7 +60,6 @@ public class VariousDocTest extends AttachmentUnitTestCase { /** * Test for https://github.com/elasticsearch/elasticsearch-mapper-attachments/issues/104 */ - @Test public void testWordDocxDocument104() throws Exception { assertParseable("issue-104.docx"); testMapper("issue-104.docx", false); @@ -68,7 +68,6 @@ public class VariousDocTest extends AttachmentUnitTestCase { /** * Test for encrypted PDF */ - @Test public void testEncryptedPDFDocument() throws Exception { assertException("encrypted.pdf"); // TODO Remove when this will be fixed in Tika. See https://issues.apache.org/jira/browse/TIKA-1548 @@ -79,7 +78,6 @@ public class VariousDocTest extends AttachmentUnitTestCase { /** * Test for HTML */ - @Test public void testHtmlDocument() throws Exception { assertParseable("htmlWithEmptyDateMeta.html"); testMapper("htmlWithEmptyDateMeta.html", false); @@ -88,7 +86,6 @@ public class VariousDocTest extends AttachmentUnitTestCase { /** * Test for XHTML */ - @Test public void testXHtmlDocument() throws Exception { assertParseable("testXHTML.html"); testMapper("testXHTML.html", false); @@ -97,7 +94,6 @@ public class VariousDocTest extends AttachmentUnitTestCase { /** * Test for TXT */ - @Test public void testTxtDocument() throws Exception { assertParseable("text-in-english.txt"); testMapper("text-in-english.txt", false); @@ -107,7 +103,6 @@ public class VariousDocTest extends AttachmentUnitTestCase { * Test for ASCIIDOC * Not yet supported by Tika: https://github.com/elasticsearch/elasticsearch-mapper-attachments/issues/29 */ - @Test public void testAsciidocDocument() throws Exception { assertParseable("asciidoc.asciidoc"); testMapper("asciidoc.asciidoc", false); diff --git a/src/test/resources/integ-tests-183.xml b/src/test/resources/integ-tests-183.xml deleted file mode 100644 index 8c9edb210c1..00000000000 --- a/src/test/resources/integ-tests-183.xml +++ /dev/null @@ -1,404 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Installing plugin @{name}... - - - - - - - - - - - - - - - - - - - - - - - - - Waiting for elasticsearch to become available on port @{port}... - - - - - - - - - - - - Waiting for elasticsearch to form a cluster of two... - - - - - - - - - - - - - - - - - - - - - - - - - - - Starting up external cluster... - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - External node started PID ${integ.pid} - - - - - - - - - - - - - - - - - - - - - - - - - Shutting down external node PID ${integ.pid} - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -