Merge pull request #185 from rmuir/fix_security_exceptions
Fix security problems in mapper-attachments.
This commit is contained in:
commit
f2ee1d79fa
|
@ -13,5 +13,7 @@
|
||||||
/.local-*-execution-hints.log
|
/.local-*-execution-hints.log
|
||||||
/eclipse-build/
|
/eclipse-build/
|
||||||
build/
|
build/
|
||||||
|
**/.local*
|
||||||
generated-resources/
|
generated-resources/
|
||||||
.gradle/
|
.gradle/
|
||||||
|
/bin/
|
||||||
|
|
|
@ -37,8 +37,9 @@ apply plugin: 'eclipse'
|
||||||
apply plugin: 'elasticsearch.esplugin'
|
apply plugin: 'elasticsearch.esplugin'
|
||||||
|
|
||||||
esplugin {
|
esplugin {
|
||||||
|
name 'mapper-attachments'
|
||||||
description 'The mapper attachments plugin adds the attachment type to Elasticsearch using Apache Tika.'
|
description 'The mapper attachments plugin adds the attachment type to Elasticsearch using Apache Tika.'
|
||||||
classname 'org.elasticsearch.plugin.mapper.attachments.MapperAttachmentsPlugin'
|
classname 'org.elasticsearch.mapper.attachments.MapperAttachmentsPlugin'
|
||||||
}
|
}
|
||||||
|
|
||||||
group = 'org.elasticsearch.plugin'
|
group = 'org.elasticsearch.plugin'
|
||||||
|
@ -60,7 +61,7 @@ repositories {
|
||||||
}
|
}
|
||||||
|
|
||||||
dependencies {
|
dependencies {
|
||||||
compile('org.apache.tika:tika-parsers:1.10') {
|
compile('org.apache.tika:tika-parsers:1.11') {
|
||||||
// TODO: is all of edu.ucar incompatibile with apache2 license? if so, we can exclude the group?
|
// TODO: is all of edu.ucar incompatibile with apache2 license? if so, we can exclude the group?
|
||||||
// Not Apache2 License compatible
|
// Not Apache2 License compatible
|
||||||
exclude group: 'edu.ucar', module: 'netcdf'
|
exclude group: 'edu.ucar', module: 'netcdf'
|
||||||
|
|
|
@ -17,32 +17,34 @@
|
||||||
* under the License.
|
* under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
package org.elasticsearch.index.mapper.attachment;
|
package org.elasticsearch.mapper.attachments;
|
||||||
|
|
||||||
import org.apache.lucene.document.Field;
|
import org.apache.lucene.document.Field;
|
||||||
import org.apache.lucene.index.IndexOptions;
|
import org.apache.lucene.index.IndexOptions;
|
||||||
import org.apache.lucene.util.Constants;
|
|
||||||
import org.apache.tika.Tika;
|
import org.apache.tika.Tika;
|
||||||
|
import org.apache.tika.exception.TikaException;
|
||||||
import org.apache.tika.language.LanguageIdentifier;
|
import org.apache.tika.language.LanguageIdentifier;
|
||||||
import org.apache.tika.metadata.Metadata;
|
import org.apache.tika.metadata.Metadata;
|
||||||
|
import org.elasticsearch.SpecialPermission;
|
||||||
import org.elasticsearch.Version;
|
import org.elasticsearch.Version;
|
||||||
import org.elasticsearch.common.collect.Iterators;
|
import org.elasticsearch.common.collect.Iterators;
|
||||||
import org.elasticsearch.common.io.stream.StreamInput;
|
import org.elasticsearch.common.io.stream.StreamInput;
|
||||||
import org.elasticsearch.common.logging.ESLogger;
|
import org.elasticsearch.common.logging.ESLogger;
|
||||||
import org.elasticsearch.common.logging.ESLoggerFactory;
|
import org.elasticsearch.common.logging.ESLoggerFactory;
|
||||||
import org.elasticsearch.common.settings.Settings;
|
import org.elasticsearch.common.settings.Settings;
|
||||||
import org.elasticsearch.common.util.CollectionUtils;
|
|
||||||
import org.elasticsearch.common.xcontent.XContentBuilder;
|
import org.elasticsearch.common.xcontent.XContentBuilder;
|
||||||
import org.elasticsearch.common.xcontent.XContentParser;
|
import org.elasticsearch.common.xcontent.XContentParser;
|
||||||
import org.elasticsearch.index.mapper.*;
|
import org.elasticsearch.index.mapper.*;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
|
import java.security.AccessController;
|
||||||
|
import java.security.PrivilegedActionException;
|
||||||
|
import java.security.PrivilegedExceptionAction;
|
||||||
import java.util.*;
|
import java.util.*;
|
||||||
|
|
||||||
import static org.elasticsearch.index.mapper.MapperBuilders.*;
|
import static org.elasticsearch.index.mapper.MapperBuilders.*;
|
||||||
import static org.elasticsearch.index.mapper.core.TypeParsers.parseMultiField;
|
import static org.elasticsearch.index.mapper.core.TypeParsers.parseMultiField;
|
||||||
import static org.elasticsearch.index.mapper.core.TypeParsers.parsePathType;
|
import static org.elasticsearch.index.mapper.core.TypeParsers.parsePathType;
|
||||||
import static org.elasticsearch.plugin.mapper.attachments.tika.TikaInstance.tika;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* <pre>
|
* <pre>
|
||||||
|
@ -482,20 +484,7 @@ public class AttachmentMapper extends FieldMapper {
|
||||||
|
|
||||||
String parsedContent;
|
String parsedContent;
|
||||||
try {
|
try {
|
||||||
Tika tika = tika();
|
parsedContent = parseWithTika(content, metadata, indexedChars);
|
||||||
if (tika == null) {
|
|
||||||
if (!ignoreErrors) {
|
|
||||||
throw new MapperParsingException("Tika can not be initialized with the current Locale [" +
|
|
||||||
Locale.getDefault().getLanguage() + "] on the current JVM [" +
|
|
||||||
Constants.JAVA_VERSION + "]");
|
|
||||||
} else {
|
|
||||||
logger.warn("Tika can not be initialized with the current Locale [{}] on the current JVM [{}]",
|
|
||||||
Locale.getDefault().getLanguage(), Constants.JAVA_VERSION);
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// Set the maximum length of strings returned by the parseToString method, -1 sets no limit
|
|
||||||
parsedContent = tika.parseToString(StreamInput.wrap(content), metadata, indexedChars);
|
|
||||||
} catch (Throwable e) {
|
} catch (Throwable e) {
|
||||||
// #18: we could ignore errors when Tika does not parse data
|
// #18: we could ignore errors when Tika does not parse data
|
||||||
if (!ignoreErrors) {
|
if (!ignoreErrors) {
|
||||||
|
@ -614,6 +603,40 @@ public class AttachmentMapper extends FieldMapper {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// singleton
|
||||||
|
private static final Tika TIKA_INSTANCE = new Tika();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* parses with tika, throwing any exception hit while parsing the document
|
||||||
|
*/
|
||||||
|
// only package private for testing!
|
||||||
|
static String parseWithTika(final byte content[], final Metadata metadata, final int limit) throws TikaException, IOException {
|
||||||
|
// check that its not unprivileged code like a script
|
||||||
|
SecurityManager sm = System.getSecurityManager();
|
||||||
|
if (sm != null) {
|
||||||
|
sm.checkPermission(new SpecialPermission());
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
return AccessController.doPrivileged(new PrivilegedExceptionAction<String>() {
|
||||||
|
@Override
|
||||||
|
public String run() throws TikaException, IOException {
|
||||||
|
return TIKA_INSTANCE.parseToString(StreamInput.wrap(content), metadata, limit);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
} catch (PrivilegedActionException e) {
|
||||||
|
// checked exception from tika: unbox it
|
||||||
|
Throwable cause = e.getCause();
|
||||||
|
if (cause instanceof TikaException) {
|
||||||
|
throw (TikaException) cause;
|
||||||
|
} else if (cause instanceof IOException) {
|
||||||
|
throw (IOException) cause;
|
||||||
|
} else {
|
||||||
|
throw new AssertionError(cause);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected void parseCreateField(ParseContext parseContext, List<Field> fields) throws IOException {
|
protected void parseCreateField(ParseContext parseContext, List<Field> fields) throws IOException {
|
||||||
|
|
|
@ -17,10 +17,9 @@
|
||||||
* under the License.
|
* under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
package org.elasticsearch.plugin.mapper.attachments;
|
package org.elasticsearch.mapper.attachments;
|
||||||
|
|
||||||
import org.elasticsearch.index.IndexService;
|
import org.elasticsearch.index.IndexService;
|
||||||
import org.elasticsearch.index.mapper.attachment.AttachmentMapper;
|
|
||||||
import org.elasticsearch.plugins.Plugin;
|
import org.elasticsearch.plugins.Plugin;
|
||||||
|
|
||||||
public class MapperAttachmentsPlugin extends Plugin {
|
public class MapperAttachmentsPlugin extends Plugin {
|
|
@ -1,53 +0,0 @@
|
||||||
/*
|
|
||||||
* Licensed to Elasticsearch under one or more contributor
|
|
||||||
* license agreements. See the NOTICE file distributed with
|
|
||||||
* this work for additional information regarding copyright
|
|
||||||
* ownership. Elasticsearch licenses this file to you under
|
|
||||||
* the Apache License, Version 2.0 (the "License"); you may
|
|
||||||
* not use this file except in compliance with the License.
|
|
||||||
* You may obtain a copy of the License at
|
|
||||||
*
|
|
||||||
* http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
*
|
|
||||||
* Unless required by applicable law or agreed to in writing,
|
|
||||||
* software distributed under the License is distributed on an
|
|
||||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
||||||
* KIND, either express or implied. See the License for the
|
|
||||||
* specific language governing permissions and limitations
|
|
||||||
* under the License.
|
|
||||||
*/
|
|
||||||
|
|
||||||
package org.elasticsearch.plugin.mapper.attachments.tika;
|
|
||||||
|
|
||||||
import org.apache.lucene.util.Constants;
|
|
||||||
|
|
||||||
import java.util.StringTokenizer;
|
|
||||||
|
|
||||||
import static java.lang.Integer.parseInt;
|
|
||||||
|
|
||||||
public class LocaleChecker {
|
|
||||||
public static int JVM_MAJOR_VERSION = 0;
|
|
||||||
public static int JVM_MINOR_VERSION = 0;
|
|
||||||
public static int JVM_PATCH_MAJOR_VERSION = 0;
|
|
||||||
public static int JVM_PATCH_MINOR_VERSION = 0;
|
|
||||||
|
|
||||||
static {
|
|
||||||
StringTokenizer st = new StringTokenizer(Constants.JVM_SPEC_VERSION, ".");
|
|
||||||
JVM_MAJOR_VERSION = parseInt(st.nextToken());
|
|
||||||
if(st.hasMoreTokens()) {
|
|
||||||
JVM_MINOR_VERSION = parseInt(st.nextToken());
|
|
||||||
}
|
|
||||||
if(st.hasMoreTokens()) {
|
|
||||||
StringTokenizer stPatch = new StringTokenizer(st.nextToken(), "_");
|
|
||||||
JVM_PATCH_MAJOR_VERSION = parseInt(stPatch.nextToken());
|
|
||||||
JVM_PATCH_MINOR_VERSION = parseInt(stPatch.nextToken());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Tika 1.8 fixed currently known Locale issues with some JVMs
|
|
||||||
*/
|
|
||||||
public static boolean isLocaleCompatible() {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -17,29 +17,10 @@
|
||||||
* under the License.
|
* under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
package org.elasticsearch.plugin.mapper.attachments.tika;
|
grant {
|
||||||
|
// TODO: fix tika not to actually install bouncy castle like this
|
||||||
|
permission java.security.SecurityPermission "putProviderProperty.BC";
|
||||||
import org.apache.tika.Tika;
|
permission java.security.SecurityPermission "insertProvider";
|
||||||
|
// TODO: fix POI XWPF to not do this: https://bz.apache.org/bugzilla/show_bug.cgi?id=58597
|
||||||
import static org.elasticsearch.plugin.mapper.attachments.tika.LocaleChecker.isLocaleCompatible;
|
permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
|
||||||
|
};
|
||||||
/**
|
|
||||||
*
|
|
||||||
*/
|
|
||||||
public class TikaInstance {
|
|
||||||
|
|
||||||
private static final Tika tika;
|
|
||||||
|
|
||||||
static {
|
|
||||||
if (isLocaleCompatible()) {
|
|
||||||
tika = new Tika();
|
|
||||||
} else {
|
|
||||||
tika = null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public static Tika tika() {
|
|
||||||
return tika;
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -17,26 +17,15 @@
|
||||||
* under the License.
|
* under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
package org.elasticsearch.index.mapper.attachment.test.unit;
|
package org.elasticsearch.mapper.attachments;
|
||||||
|
|
||||||
import org.elasticsearch.Version;
|
import org.elasticsearch.Version;
|
||||||
import org.elasticsearch.cluster.metadata.IndexMetaData;
|
import org.elasticsearch.cluster.metadata.IndexMetaData;
|
||||||
import org.elasticsearch.common.settings.Settings;
|
import org.elasticsearch.common.settings.Settings;
|
||||||
import org.elasticsearch.test.ESTestCase;
|
import org.elasticsearch.test.ESTestCase;
|
||||||
import org.junit.Before;
|
import org.junit.Before;
|
||||||
import org.junit.BeforeClass;
|
|
||||||
|
|
||||||
import static org.elasticsearch.index.mapper.attachment.test.MapperTestUtils.assumeCorrectLocale;
|
|
||||||
|
|
||||||
public class AttachmentUnitTestCase extends ESTestCase {
|
public class AttachmentUnitTestCase extends ESTestCase {
|
||||||
/**
|
|
||||||
* We can have issues with some JVMs and Locale
|
|
||||||
* See https://github.com/elasticsearch/elasticsearch-mapper-attachments/issues/105
|
|
||||||
*/
|
|
||||||
@BeforeClass
|
|
||||||
public static void checkLocale() {
|
|
||||||
assumeCorrectLocale();
|
|
||||||
}
|
|
||||||
|
|
||||||
protected Settings testSettings;
|
protected Settings testSettings;
|
||||||
|
|
|
@ -17,16 +17,14 @@
|
||||||
* under the License.
|
* under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
package org.elasticsearch.index.mapper.attachment.test.unit;
|
package org.elasticsearch.mapper.attachments;
|
||||||
|
|
||||||
import org.elasticsearch.common.settings.Settings;
|
import org.elasticsearch.common.settings.Settings;
|
||||||
import org.elasticsearch.index.mapper.DocumentMapper;
|
import org.elasticsearch.index.mapper.DocumentMapper;
|
||||||
import org.elasticsearch.index.mapper.DocumentMapperParser;
|
import org.elasticsearch.index.mapper.DocumentMapperParser;
|
||||||
import org.elasticsearch.index.mapper.attachment.AttachmentMapper;
|
|
||||||
import org.elasticsearch.index.mapper.attachment.test.MapperTestUtils;
|
|
||||||
import org.elasticsearch.index.mapper.core.StringFieldMapper;
|
import org.elasticsearch.index.mapper.core.StringFieldMapper;
|
||||||
|
import org.elasticsearch.mapper.attachments.AttachmentMapper;
|
||||||
import org.junit.Before;
|
import org.junit.Before;
|
||||||
import org.junit.Test;
|
|
||||||
|
|
||||||
import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath;
|
import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath;
|
||||||
import static org.hamcrest.Matchers.instanceOf;
|
import static org.hamcrest.Matchers.instanceOf;
|
|
@ -17,7 +17,7 @@
|
||||||
* under the License.
|
* under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
package org.elasticsearch.index.mapper.attachment.test.unit;
|
package org.elasticsearch.mapper.attachments;
|
||||||
|
|
||||||
import org.elasticsearch.common.bytes.BytesReference;
|
import org.elasticsearch.common.bytes.BytesReference;
|
||||||
import org.elasticsearch.common.settings.Settings;
|
import org.elasticsearch.common.settings.Settings;
|
||||||
|
@ -25,9 +25,7 @@ import org.elasticsearch.index.mapper.DocumentMapper;
|
||||||
import org.elasticsearch.index.mapper.DocumentMapperParser;
|
import org.elasticsearch.index.mapper.DocumentMapperParser;
|
||||||
import org.elasticsearch.index.mapper.MapperParsingException;
|
import org.elasticsearch.index.mapper.MapperParsingException;
|
||||||
import org.elasticsearch.index.mapper.ParseContext;
|
import org.elasticsearch.index.mapper.ParseContext;
|
||||||
import org.elasticsearch.index.mapper.attachment.AttachmentMapper;
|
import org.elasticsearch.mapper.attachments.AttachmentMapper;
|
||||||
import org.elasticsearch.index.mapper.attachment.test.MapperTestUtils;
|
|
||||||
import org.junit.Test;
|
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
|
|
||||||
|
@ -41,7 +39,7 @@ import static org.hamcrest.Matchers.*;
|
||||||
* Note that we have converted /org/elasticsearch/index/mapper/xcontent/testContentLength.txt
|
* Note that we have converted /org/elasticsearch/index/mapper/xcontent/testContentLength.txt
|
||||||
* to a /org/elasticsearch/index/mapper/xcontent/encrypted.pdf with password `12345678`.
|
* to a /org/elasticsearch/index/mapper/xcontent/encrypted.pdf with password `12345678`.
|
||||||
*/
|
*/
|
||||||
public class EncryptedDocMapperTest extends AttachmentUnitTestCase {
|
public class EncryptedDocMapperTests extends AttachmentUnitTestCase {
|
||||||
|
|
||||||
public void testMultipleDocsEncryptedLast() throws IOException {
|
public void testMultipleDocsEncryptedLast() throws IOException {
|
||||||
DocumentMapperParser mapperParser = MapperTestUtils.newMapperService(createTempDir(), Settings.EMPTY).documentMapperParser();
|
DocumentMapperParser mapperParser = MapperTestUtils.newMapperService(createTempDir(), Settings.EMPTY).documentMapperParser();
|
||||||
|
@ -127,8 +125,10 @@ public class EncryptedDocMapperTest extends AttachmentUnitTestCase {
|
||||||
docMapper.parse("person", "person", "1", json);
|
docMapper.parse("person", "person", "1", json);
|
||||||
fail("Expected doc parsing exception");
|
fail("Expected doc parsing exception");
|
||||||
} catch (MapperParsingException e) {
|
} catch (MapperParsingException e) {
|
||||||
// TODO: check the error message...getting security problems atm
|
if (e.getMessage() == null || e.getMessage().contains("is encrypted") == false) {
|
||||||
//assertTrue(e.getMessage(), e.getMessage().contains())
|
// wrong exception
|
||||||
|
throw e;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -17,18 +17,16 @@
|
||||||
* under the License.
|
* under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
package org.elasticsearch.index.mapper.attachment.test.unit;
|
package org.elasticsearch.mapper.attachments;
|
||||||
|
|
||||||
import org.elasticsearch.common.settings.Settings;
|
import org.elasticsearch.common.settings.Settings;
|
||||||
import org.elasticsearch.common.xcontent.XContentBuilder;
|
import org.elasticsearch.common.xcontent.XContentBuilder;
|
||||||
import org.elasticsearch.index.mapper.DocumentMapper;
|
import org.elasticsearch.index.mapper.DocumentMapper;
|
||||||
import org.elasticsearch.index.mapper.DocumentMapperParser;
|
import org.elasticsearch.index.mapper.DocumentMapperParser;
|
||||||
import org.elasticsearch.index.mapper.ParseContext;
|
import org.elasticsearch.index.mapper.ParseContext;
|
||||||
import org.elasticsearch.index.mapper.attachment.AttachmentMapper;
|
|
||||||
import org.elasticsearch.index.mapper.attachment.test.MapperTestUtils;
|
|
||||||
import org.elasticsearch.index.mapper.core.StringFieldMapper;
|
import org.elasticsearch.index.mapper.core.StringFieldMapper;
|
||||||
|
import org.elasticsearch.mapper.attachments.AttachmentMapper;
|
||||||
import org.junit.Before;
|
import org.junit.Before;
|
||||||
import org.junit.Test;
|
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
|
|
|
@ -17,12 +17,12 @@
|
||||||
* under the License.
|
* under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
package org.elasticsearch.index.mapper.attachment.test.integration;
|
package org.elasticsearch.mapper.attachments;
|
||||||
|
|
||||||
import com.carrotsearch.randomizedtesting.annotations.Name;
|
import com.carrotsearch.randomizedtesting.annotations.Name;
|
||||||
import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
|
import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
|
||||||
import org.elasticsearch.common.settings.Settings;
|
import org.elasticsearch.common.settings.Settings;
|
||||||
import org.elasticsearch.plugin.mapper.attachments.MapperAttachmentsPlugin;
|
import org.elasticsearch.mapper.attachments.MapperAttachmentsPlugin;
|
||||||
import org.elasticsearch.test.rest.ESRestTestCase;
|
import org.elasticsearch.test.rest.ESRestTestCase;
|
||||||
import org.elasticsearch.test.rest.RestTestCandidate;
|
import org.elasticsearch.test.rest.RestTestCandidate;
|
||||||
import org.elasticsearch.test.rest.parser.RestTestParseException;
|
import org.elasticsearch.test.rest.parser.RestTestParseException;
|
|
@ -17,9 +17,8 @@
|
||||||
* under the License.
|
* under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
package org.elasticsearch.index.mapper.attachment.test;
|
package org.elasticsearch.mapper.attachments;
|
||||||
|
|
||||||
import org.apache.lucene.util.Constants;
|
|
||||||
import org.elasticsearch.Version;
|
import org.elasticsearch.Version;
|
||||||
import org.elasticsearch.cluster.metadata.IndexMetaData;
|
import org.elasticsearch.cluster.metadata.IndexMetaData;
|
||||||
import org.elasticsearch.common.settings.Settings;
|
import org.elasticsearch.common.settings.Settings;
|
||||||
|
@ -35,12 +34,8 @@ import org.elasticsearch.test.IndexSettingsModule;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.nio.file.Path;
|
import java.nio.file.Path;
|
||||||
import java.util.Collections;
|
import java.util.Collections;
|
||||||
import java.util.Locale;
|
|
||||||
|
|
||||||
import static com.carrotsearch.randomizedtesting.RandomizedTest.assumeTrue;
|
class MapperTestUtils {
|
||||||
import static org.elasticsearch.plugin.mapper.attachments.tika.LocaleChecker.isLocaleCompatible;
|
|
||||||
|
|
||||||
public class MapperTestUtils {
|
|
||||||
|
|
||||||
public static MapperService newMapperService(Path tempDir, Settings indexSettings) throws IOException {
|
public static MapperService newMapperService(Path tempDir, Settings indexSettings) throws IOException {
|
||||||
Settings nodeSettings = Settings.builder()
|
Settings nodeSettings = Settings.builder()
|
||||||
|
@ -55,14 +50,4 @@ public class MapperTestUtils {
|
||||||
SimilarityService similarityService = new SimilarityService(idxSettings, Collections.emptyMap());
|
SimilarityService similarityService = new SimilarityService(idxSettings, Collections.emptyMap());
|
||||||
return new MapperService(idxSettings, analysisService, similarityService);
|
return new MapperService(idxSettings, analysisService, similarityService);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* We can have issues with some JVMs and Locale
|
|
||||||
* See https://github.com/elasticsearch/elasticsearch-mapper-attachments/issues/105
|
|
||||||
*/
|
|
||||||
public static void assumeCorrectLocale() {
|
|
||||||
assumeTrue("Current Locale language " + Locale.getDefault().getLanguage() +" could cause an error with Java " +
|
|
||||||
Constants.JAVA_VERSION, isLocaleCompatible());
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
}
|
|
@ -17,7 +17,7 @@
|
||||||
* under the License.
|
* under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
package org.elasticsearch.index.mapper.attachment.test.unit;
|
package org.elasticsearch.mapper.attachments;
|
||||||
|
|
||||||
import org.elasticsearch.common.bytes.BytesReference;
|
import org.elasticsearch.common.bytes.BytesReference;
|
||||||
import org.elasticsearch.common.settings.Settings;
|
import org.elasticsearch.common.settings.Settings;
|
||||||
|
@ -25,9 +25,7 @@ import org.elasticsearch.index.mapper.DocumentMapper;
|
||||||
import org.elasticsearch.index.mapper.DocumentMapperParser;
|
import org.elasticsearch.index.mapper.DocumentMapperParser;
|
||||||
import org.elasticsearch.index.mapper.MapperParsingException;
|
import org.elasticsearch.index.mapper.MapperParsingException;
|
||||||
import org.elasticsearch.index.mapper.ParseContext;
|
import org.elasticsearch.index.mapper.ParseContext;
|
||||||
import org.elasticsearch.index.mapper.attachment.AttachmentMapper;
|
import org.elasticsearch.mapper.attachments.AttachmentMapper;
|
||||||
import org.elasticsearch.index.mapper.attachment.test.MapperTestUtils;
|
|
||||||
import org.junit.Test;
|
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
|
|
||||||
|
@ -39,7 +37,7 @@ import static org.hamcrest.Matchers.*;
|
||||||
/**
|
/**
|
||||||
* Test for https://github.com/elasticsearch/elasticsearch-mapper-attachments/issues/38
|
* Test for https://github.com/elasticsearch/elasticsearch-mapper-attachments/issues/38
|
||||||
*/
|
*/
|
||||||
public class MetadataMapperTest extends AttachmentUnitTestCase {
|
public class MetadataMapperTests extends AttachmentUnitTestCase {
|
||||||
|
|
||||||
protected void checkMeta(String filename, Settings otherSettings, Long expectedDate, Long expectedLength) throws IOException {
|
protected void checkMeta(String filename, Settings otherSettings, Long expectedDate, Long expectedLength) throws IOException {
|
||||||
Settings settings = Settings.builder()
|
Settings settings = Settings.builder()
|
||||||
|
@ -95,8 +93,8 @@ public class MetadataMapperTest extends AttachmentUnitTestCase {
|
||||||
public void testWithEmptyDate() throws Exception {
|
public void testWithEmptyDate() throws Exception {
|
||||||
try {
|
try {
|
||||||
checkMeta("htmlWithEmptyDateMeta.html", Settings.builder().put("index.mapping.attachment.ignore_errors", false).build(), null, null);
|
checkMeta("htmlWithEmptyDateMeta.html", Settings.builder().put("index.mapping.attachment.ignore_errors", false).build(), null, null);
|
||||||
} catch (MapperParsingException e) {
|
} catch (MapperParsingException expected) {
|
||||||
throw e;
|
assertTrue(expected.getMessage().contains("failed to parse"));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -17,7 +17,7 @@
|
||||||
* under the License.
|
* under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
package org.elasticsearch.index.mapper.attachment.test.unit;
|
package org.elasticsearch.mapper.attachments;
|
||||||
|
|
||||||
import org.elasticsearch.common.Base64;
|
import org.elasticsearch.common.Base64;
|
||||||
import org.elasticsearch.common.settings.Settings;
|
import org.elasticsearch.common.settings.Settings;
|
||||||
|
@ -26,14 +26,12 @@ import org.elasticsearch.index.mapper.DocumentMapper;
|
||||||
import org.elasticsearch.index.mapper.DocumentMapperParser;
|
import org.elasticsearch.index.mapper.DocumentMapperParser;
|
||||||
import org.elasticsearch.index.mapper.MapperService;
|
import org.elasticsearch.index.mapper.MapperService;
|
||||||
import org.elasticsearch.index.mapper.ParsedDocument;
|
import org.elasticsearch.index.mapper.ParsedDocument;
|
||||||
import org.elasticsearch.index.mapper.attachment.AttachmentMapper;
|
|
||||||
import org.elasticsearch.index.mapper.attachment.test.MapperTestUtils;
|
|
||||||
import org.elasticsearch.index.mapper.core.DateFieldMapper;
|
import org.elasticsearch.index.mapper.core.DateFieldMapper;
|
||||||
import org.elasticsearch.index.mapper.core.StringFieldMapper;
|
import org.elasticsearch.index.mapper.core.StringFieldMapper;
|
||||||
|
import org.elasticsearch.mapper.attachments.AttachmentMapper;
|
||||||
import org.elasticsearch.threadpool.ThreadPool;
|
import org.elasticsearch.threadpool.ThreadPool;
|
||||||
import org.junit.After;
|
import org.junit.After;
|
||||||
import org.junit.Before;
|
import org.junit.Before;
|
||||||
import org.junit.Test;
|
|
||||||
|
|
||||||
import java.nio.charset.StandardCharsets;
|
import java.nio.charset.StandardCharsets;
|
||||||
|
|
|
@ -17,7 +17,7 @@
|
||||||
* under the License.
|
* under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
package org.elasticsearch.index.mapper.attachment.test.unit;
|
package org.elasticsearch.mapper.attachments;
|
||||||
|
|
||||||
import org.elasticsearch.Version;
|
import org.elasticsearch.Version;
|
||||||
import org.elasticsearch.cluster.metadata.IndexMetaData;
|
import org.elasticsearch.cluster.metadata.IndexMetaData;
|
||||||
|
@ -26,8 +26,7 @@ import org.elasticsearch.common.settings.Settings;
|
||||||
import org.elasticsearch.index.mapper.DocumentMapper;
|
import org.elasticsearch.index.mapper.DocumentMapper;
|
||||||
import org.elasticsearch.index.mapper.DocumentMapperParser;
|
import org.elasticsearch.index.mapper.DocumentMapperParser;
|
||||||
import org.elasticsearch.index.mapper.ParseContext;
|
import org.elasticsearch.index.mapper.ParseContext;
|
||||||
import org.elasticsearch.index.mapper.attachment.AttachmentMapper;
|
import org.elasticsearch.mapper.attachments.AttachmentMapper;
|
||||||
import org.elasticsearch.index.mapper.attachment.test.MapperTestUtils;
|
|
||||||
|
|
||||||
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
|
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
|
||||||
import static org.elasticsearch.test.StreamsUtils.copyToBytesFromClasspath;
|
import static org.elasticsearch.test.StreamsUtils.copyToBytesFromClasspath;
|
|
@ -17,7 +17,7 @@
|
||||||
* under the License.
|
* under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
package org.elasticsearch.index.mapper.attachment.test.standalone;
|
package org.elasticsearch.mapper.attachments;
|
||||||
|
|
||||||
import org.apache.commons.cli.CommandLine;
|
import org.apache.commons.cli.CommandLine;
|
||||||
import org.elasticsearch.common.bytes.BytesReference;
|
import org.elasticsearch.common.bytes.BytesReference;
|
||||||
|
@ -32,8 +32,7 @@ import org.elasticsearch.env.Environment;
|
||||||
import org.elasticsearch.index.mapper.DocumentMapper;
|
import org.elasticsearch.index.mapper.DocumentMapper;
|
||||||
import org.elasticsearch.index.mapper.DocumentMapperParser;
|
import org.elasticsearch.index.mapper.DocumentMapperParser;
|
||||||
import org.elasticsearch.index.mapper.ParseContext;
|
import org.elasticsearch.index.mapper.ParseContext;
|
||||||
import org.elasticsearch.index.mapper.attachment.AttachmentMapper;
|
import org.elasticsearch.mapper.attachments.AttachmentMapper;
|
||||||
import org.elasticsearch.index.mapper.attachment.test.MapperTestUtils;
|
|
||||||
|
|
||||||
import java.io.FileNotFoundException;
|
import java.io.FileNotFoundException;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
|
@ -17,25 +17,23 @@
|
||||||
* under the License.
|
* under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
package org.elasticsearch.index.mapper.attachment.test.unit;
|
package org.elasticsearch.mapper.attachments;
|
||||||
|
|
||||||
import org.apache.tika.Tika;
|
import org.apache.tika.io.IOUtils;
|
||||||
|
import org.apache.tika.metadata.Metadata;
|
||||||
import org.elasticsearch.common.bytes.BytesReference;
|
import org.elasticsearch.common.bytes.BytesReference;
|
||||||
import org.elasticsearch.common.settings.Settings;
|
import org.elasticsearch.common.settings.Settings;
|
||||||
import org.elasticsearch.index.mapper.DocumentMapper;
|
import org.elasticsearch.index.mapper.DocumentMapper;
|
||||||
import org.elasticsearch.index.mapper.DocumentMapperParser;
|
import org.elasticsearch.index.mapper.DocumentMapperParser;
|
||||||
import org.elasticsearch.index.mapper.ParseContext;
|
import org.elasticsearch.index.mapper.ParseContext;
|
||||||
import org.elasticsearch.index.mapper.attachment.AttachmentMapper;
|
import org.elasticsearch.mapper.attachments.AttachmentMapper;
|
||||||
import org.elasticsearch.index.mapper.attachment.test.MapperTestUtils;
|
|
||||||
import org.junit.Before;
|
import org.junit.Before;
|
||||||
import org.junit.Test;
|
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.io.InputStream;
|
import java.io.InputStream;
|
||||||
|
|
||||||
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
|
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
|
||||||
import static org.elasticsearch.index.mapper.attachment.AttachmentMapper.FieldNames.*;
|
import static org.elasticsearch.mapper.attachments.AttachmentMapper.FieldNames.*;
|
||||||
import static org.elasticsearch.plugin.mapper.attachments.tika.TikaInstance.tika;
|
|
||||||
import static org.elasticsearch.test.StreamsUtils.copyToBytesFromClasspath;
|
import static org.elasticsearch.test.StreamsUtils.copyToBytesFromClasspath;
|
||||||
import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath;
|
import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath;
|
||||||
import static org.hamcrest.Matchers.isEmptyOrNullString;
|
import static org.hamcrest.Matchers.isEmptyOrNullString;
|
||||||
|
@ -44,7 +42,7 @@ import static org.hamcrest.Matchers.not;
|
||||||
/**
|
/**
|
||||||
* Test for different documents
|
* Test for different documents
|
||||||
*/
|
*/
|
||||||
public class VariousDocTest extends AttachmentUnitTestCase {
|
public class VariousDocTests extends AttachmentUnitTestCase {
|
||||||
|
|
||||||
protected DocumentMapper docMapper;
|
protected DocumentMapper docMapper;
|
||||||
|
|
||||||
|
@ -69,7 +67,7 @@ public class VariousDocTest extends AttachmentUnitTestCase {
|
||||||
* Test for encrypted PDF
|
* Test for encrypted PDF
|
||||||
*/
|
*/
|
||||||
public void testEncryptedPDFDocument() throws Exception {
|
public void testEncryptedPDFDocument() throws Exception {
|
||||||
assertException("encrypted.pdf");
|
assertException("encrypted.pdf", "is encrypted");
|
||||||
// TODO Remove when this will be fixed in Tika. See https://issues.apache.org/jira/browse/TIKA-1548
|
// TODO Remove when this will be fixed in Tika. See https://issues.apache.org/jira/browse/TIKA-1548
|
||||||
System.clearProperty("sun.font.fontmanager");
|
System.clearProperty("sun.font.fontmanager");
|
||||||
testMapper("encrypted.pdf", true);
|
testMapper("encrypted.pdf", true);
|
||||||
|
@ -108,24 +106,25 @@ public class VariousDocTest extends AttachmentUnitTestCase {
|
||||||
testMapper("asciidoc.asciidoc", false);
|
testMapper("asciidoc.asciidoc", false);
|
||||||
}
|
}
|
||||||
|
|
||||||
void assertException(String filename) {
|
void assertException(String filename, String expectedMessage) throws Exception {
|
||||||
Tika tika = tika();
|
try (InputStream is = VariousDocTests.class.getResourceAsStream("/org/elasticsearch/index/mapper/attachment/test/sample-files/" + filename)) {
|
||||||
assumeTrue("Tika has been disabled. Ignoring test...", tika != null);
|
byte bytes[] = IOUtils.toByteArray(is);
|
||||||
|
AttachmentMapper.parseWithTika(bytes, new Metadata(), -1);
|
||||||
try (InputStream is = VariousDocTest.class.getResourceAsStream("/org/elasticsearch/index/mapper/attachment/test/sample-files/" + filename)) {
|
|
||||||
tika.parseToString(is);
|
|
||||||
fail("expected exception");
|
fail("expected exception");
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
// expected. TODO: check message
|
if (e.getMessage() != null && e.getMessage().contains(expectedMessage)) {
|
||||||
|
// ok
|
||||||
|
} else {
|
||||||
|
// unexpected
|
||||||
|
throw e;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
protected void assertParseable(String filename) throws Exception {
|
protected void assertParseable(String filename) throws Exception {
|
||||||
Tika tika = tika();
|
try (InputStream is = VariousDocTests.class.getResourceAsStream("/org/elasticsearch/index/mapper/attachment/test/sample-files/" + filename)) {
|
||||||
assumeTrue("Tika has been disabled. Ignoring test...", tika != null);
|
byte bytes[] = IOUtils.toByteArray(is);
|
||||||
|
String parsedContent = AttachmentMapper.parseWithTika(bytes, new Metadata(), -1);
|
||||||
try (InputStream is = VariousDocTest.class.getResourceAsStream("/org/elasticsearch/index/mapper/attachment/test/sample-files/" + filename)) {
|
|
||||||
String parsedContent = tika.parseToString(is);
|
|
||||||
assertThat(parsedContent, not(isEmptyOrNullString()));
|
assertThat(parsedContent, not(isEmptyOrNullString()));
|
||||||
logger.debug("extracted content: {}", parsedContent);
|
logger.debug("extracted content: {}", parsedContent);
|
||||||
}
|
}
|
Loading…
Reference in New Issue