honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[build] Update to elasticsearch 2.0.0 

Fix compiling issues.
Also fix test issues.

Need now to move old integration tests to RESTIT

Closes #148.
This commit is contained in:
David Pilato 2015-08-19 20:38:00 +02:00
parent 5af2a4d63d
commit b0567d7c18
8 changed files with 41 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
pom.xml
View File

@ -7,7 +7,6 @@
<groupId>org.elasticsearch</groupId>
<artifactId>elasticsearch-mapper-attachments</artifactId>
<version>3.0.0-SNAPSHOT</version>
<packaging>jar</packaging>
<name>Elasticsearch Mapper Attachment plugin</name>
<description>The mapper attachments plugin adds the attachment type to Elasticsearch using Apache Tika.</description>
<url>https://github.com/elastic/elasticsearch-mapper-attachments/</url>
@ -26,7 +25,7 @@
</scm>
<parent>
<groupId>org.elasticsearch</groupId>
<groupId>org.elasticsearch.plugin</groupId>
<artifactId>elasticsearch-plugin</artifactId>
<version>2.0.0-SNAPSHOT</version>
</parent>
@ -34,9 +33,18 @@
<properties>
<!-- If we need to define any specific property -->
<elasticsearch.version>2.0.0-SNAPSHOT</elasticsearch.version>
<elasticsearch.plugin.classname>org.elasticsearch.plugin.mapper.attachments.MapperAttachmentsPlugin</elasticsearch.plugin.classname>
</properties>
<dependencies>
<dependency>
<groupId>org.elasticsearch.distribution.fully-loaded</groupId>
<artifactId>elasticsearch</artifactId>
<version>${elasticsearch.version}</version>
<type>pom</type>
<scope>test</scope>
</dependency>
<!-- Tika -->
<dependency>
<groupId>org.apache.tika</groupId>

15
src/main/java/org/elasticsearch/index/mapper/attachment/AttachmentMapper.java
View File

@ -20,15 +20,12 @@
package org.elasticsearch.index.mapper.attachment;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.util.CollectionUtil;
import org.apache.lucene.util.Constants;
import org.apache.tika.Tika;
import org.apache.tika.language.LanguageIdentifier;
import org.apache.tika.metadata.Metadata;
import org.elasticsearch.Version;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.logging.ESLoggerFactory;
@ -36,9 +33,7 @@ import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.CollectionUtils;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.fielddata.FieldDataType;
import org.elasticsearch.index.mapper.*;
import org.elasticsearch.index.mapper.core.AbstractFieldMapper;
import java.io.IOException;
import java.util.*;
@ -67,7 +62,7 @@ import static org.elasticsearch.plugin.mapper.attachments.tika.TikaInstance.tika
* _content_length = Specify the maximum amount of characters to extract from the attachment. If not specified, then the default for
* tika is 100,000 characters. Caution is required when setting large values as this can cause memory issues.
*/
public class AttachmentMapper extends AbstractFieldMapper {
public class AttachmentMapper extends FieldMapper {
private static ESLogger logger = ESLoggerFactory.getLogger("mapper.attachment");
@ -115,7 +110,7 @@ public class AttachmentMapper extends AbstractFieldMapper {
}
}
public static class Builder extends AbstractFieldMapper.Builder<Builder, AttachmentMapper> {
public static class Builder extends FieldMapper.Builder<Builder, AttachmentMapper> {
private ContentPath.Type pathType = Defaults.PATH_TYPE;
@ -207,10 +202,10 @@ public class AttachmentMapper extends AbstractFieldMapper {
FieldMapper contentMapper;
if (context.indexCreatedVersion().before(Version.V_2_0_0)) {
// old behavior, we need the content to be indexed under the attachment field name
if (contentBuilder instanceof AbstractFieldMapper.Builder == false) {
if (contentBuilder instanceof FieldMapper.Builder == false) {
throw new IllegalStateException("content field for attachment must be a field mapper");
}
((AbstractFieldMapper.Builder)contentBuilder).indexName(name);
((FieldMapper.Builder)contentBuilder).indexName(name);
contentBuilder.name = name + "." + FieldNames.CONTENT;
contentMapper = (FieldMapper) contentBuilder.build(context);
context.path().add(name);
@ -329,7 +324,7 @@ public class AttachmentMapper extends AbstractFieldMapper {
Map<String, Object> propNode = (Map<String, Object>) entry1.getValue();
Mapper.Builder<?, ?> mapperBuilder = findMapperBuilder(propNode, propName, parserContext);
if (parseMultiField((AbstractFieldMapper.Builder) mapperBuilder, fieldName, parserContext, propName, propNode)) {
if (parseMultiField((FieldMapper.Builder) mapperBuilder, fieldName, parserContext, propName, propNode)) {
fieldsIterator.remove();
} else if (propName.equals(name) && parserContext.indexVersionCreated().before(Version.V_2_0_0)) {
builder.content(mapperBuilder);

23
src/test/java/org/elasticsearch/index/mapper/attachment/test/MapperTestUtils.java
View File

@ -32,17 +32,12 @@ import org.elasticsearch.index.Index;
import org.elasticsearch.index.IndexNameModule;
import org.elasticsearch.index.analysis.AnalysisModule;
import org.elasticsearch.index.analysis.AnalysisService;
import org.elasticsearch.index.fielddata.IndexFieldDataService;
import org.elasticsearch.index.mapper.DocumentMapperParser;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.settings.IndexSettingsModule;
import org.elasticsearch.index.similarity.SimilarityLookupService;
import org.elasticsearch.indices.analysis.IndicesAnalysisModule;
import org.elasticsearch.indices.analysis.IndicesAnalysisService;
import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache;
import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCacheListener;
import org.elasticsearch.threadpool.ThreadPool;
import java.nio.file.Path;
import java.util.Locale;
@ -52,23 +47,17 @@ import static org.elasticsearch.plugin.mapper.attachments.tika.LocaleChecker.isL
public class MapperTestUtils {
public static MapperService newMapperService(Path tempDir, ThreadPool testingThreadPool) {
public static MapperService newMapperService(Path tempDir) {
return newMapperService(new Index("test"), Settings.builder()
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
.put("path.home", tempDir)
.build(), testingThreadPool);
.build());
}
public static MapperService newMapperService(Index index, Settings indexSettings, ThreadPool testingThreadPool) {
NoneCircuitBreakerService circuitBreakerService = new NoneCircuitBreakerService();
return new MapperService(index,
public static MapperService newMapperService(Index index, Settings indexSettings) {
return new MapperService(index,
indexSettings,
newAnalysisService(indexSettings),
new IndexFieldDataService(index, Settings.Builder.EMPTY_SETTINGS,
new IndicesFieldDataCache(Settings.Builder.EMPTY_SETTINGS,
new IndicesFieldDataCacheListener(circuitBreakerService),
testingThreadPool),
circuitBreakerService),
newSimilarityLookupService(indexSettings),
null);
}
@ -106,8 +95,8 @@ public class MapperTestUtils {
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
.put(settings)
.build();
MapperService mapperService = new MapperService(new Index("test"), forcedSettings, newAnalysisService(forcedSettings), null, newSimilarityLookupService(forcedSettings), null);
return new DocumentMapperParser(new Index("test"), forcedSettings, mapperService, MapperTestUtils.newAnalysisService(forcedSettings), null, null);
MapperService mapperService = new MapperService(new Index("test"), forcedSettings, newAnalysisService(forcedSettings), newSimilarityLookupService(forcedSettings), null);
return new DocumentMapperParser(forcedSettings, mapperService, MapperTestUtils.newAnalysisService(forcedSettings), null, null);
}
/**

6
src/test/java/org/elasticsearch/index/mapper/attachment/test/integration/AttachmentIntegrationTestCase.java
View File

@ -19,15 +19,15 @@
package org.elasticsearch.index.mapper.attachment.test.integration;
import org.elasticsearch.test.ElasticsearchIntegrationTest;
import org.elasticsearch.test.ESIntegTestCase;
import org.hamcrest.Matcher;
import org.junit.BeforeClass;
import static org.elasticsearch.plugin.mapper.attachments.tika.LocaleChecker.isLocaleCompatible;
import static org.hamcrest.Matchers.not;
@ElasticsearchIntegrationTest.ClusterScope(scope = ElasticsearchIntegrationTest.Scope.SUITE)
public class AttachmentIntegrationTestCase extends ElasticsearchIntegrationTest {
@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.SUITE)
public class AttachmentIntegrationTestCase extends ESIntegTestCase {
protected static boolean expectError;

11
src/test/java/org/elasticsearch/index/mapper/attachment/test/integration/EncryptedAttachmentIntegrationTests.java
View File

@ -22,22 +22,21 @@ package org.elasticsearch.index.mapper.attachment.test.integration;
import org.elasticsearch.action.count.CountResponse;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.mapper.MapperParsingException;
import org.elasticsearch.plugins.PluginsService;
import org.elasticsearch.plugin.mapper.attachments.MapperAttachmentsPlugin;
import org.junit.Test;
import org.apache.lucene.util.LuceneTestCase.Slow;
import static org.elasticsearch.client.Requests.putMappingRequest;
import static org.elasticsearch.common.io.Streams.copyToBytesFromClasspath;
import static org.elasticsearch.common.io.Streams.copyToStringFromClasspath;
import static org.elasticsearch.common.settings.Settings.settingsBuilder;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.elasticsearch.index.query.QueryBuilders.queryStringQuery;
import static org.elasticsearch.test.StreamsUtils.copyToBytesFromClasspath;
import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath;
import static org.hamcrest.Matchers.equalTo;
/**
* Test case for issue https://github.com/elasticsearch/elasticsearch-mapper-attachments/issues/18
* TODO Change this to a real IT
*/
@Slow
public class EncryptedAttachmentIntegrationTests extends AttachmentIntegrationTestCase {
private boolean ignore_errors = true;
@ -45,7 +44,7 @@ public class EncryptedAttachmentIntegrationTests extends AttachmentIntegrationTe
protected Settings nodeSettings(int nodeOrdinal) {
return Settings.builder()
.put(super.nodeSettings(nodeOrdinal))
.put("plugins." + PluginsService.LOAD_PLUGIN_FROM_CLASSPATH, true)
.put("plugin.types", MapperAttachmentsPlugin.class.getName())
.build();
}

17
src/test/java/org/elasticsearch/index/mapper/attachment/test/integration/SimpleAttachmentIntegrationTests.java
View File

@ -24,32 +24,29 @@ import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.text.Text;
import org.elasticsearch.index.mapper.MapperParsingException;
import org.elasticsearch.plugins.PluginsService;
import org.elasticsearch.plugin.mapper.attachments.MapperAttachmentsPlugin;
import org.elasticsearch.search.highlight.HighlightField;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
import org.apache.lucene.util.LuceneTestCase.Slow;
import static org.elasticsearch.client.Requests.putMappingRequest;
import static org.elasticsearch.common.io.Streams.copyToBytesFromClasspath;
import static org.elasticsearch.common.io.Streams.copyToStringFromClasspath;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
import static org.elasticsearch.index.query.QueryBuilders.queryStringQuery;
import static org.elasticsearch.test.StreamsUtils.copyToBytesFromClasspath;
import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath;
import static org.hamcrest.Matchers.*;
/**
*
* TODO Change this to real IT
*/
@Slow
public class SimpleAttachmentIntegrationTests extends AttachmentIntegrationTestCase {
@Override
protected Settings nodeSettings(int nodeOrdinal) {
return Settings.builder()
.put(super.nodeSettings(nodeOrdinal))
.put("plugins." + PluginsService.LOAD_PLUGIN_FROM_CLASSPATH, true)
.put("plugin.types", MapperAttachmentsPlugin.class.getName())
.build();
}
@ -164,7 +161,8 @@ public class SimpleAttachmentIntegrationTests extends AttachmentIntegrationTestC
* As for now, we don't support a global `copy_to` property for `attachment` type.
* So this test is failing.
*/
@Test @Ignore
/*
@Test
public void testCopyTo() throws Exception {
String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/attachment/test/integration/simple/copy-to.json");
byte[] txt = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/attachment/test/sample-files/text-in-english.txt");
@ -180,6 +178,7 @@ public class SimpleAttachmentIntegrationTests extends AttachmentIntegrationTestC
countResponse = client().prepareCount("test").setQuery(queryStringQuery("Queen").defaultField("copy")).execute().get();
assertThatWithError(countResponse.getCount(), equalTo(1l));
}
*/
@Test
public void testHighlightAttachment() throws Exception {

4
src/test/java/org/elasticsearch/index/mapper/attachment/test/standalone/StandaloneRunner.java
View File

@ -45,8 +45,8 @@ import java.util.Locale;
import static org.elasticsearch.common.cli.CliToolConfig.Builder.cmd;
import static org.elasticsearch.common.cli.CliToolConfig.Builder.option;
import static org.elasticsearch.common.io.Streams.copy;
import static org.elasticsearch.common.io.Streams.copyToStringFromClasspath;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath;
/**
* This class provides a simple main class which can be used to test what is extracted from a given binary file.
@ -109,7 +109,7 @@ public class StandaloneRunner extends CliTool {
BytesReference json = builder.endObject().endObject().bytes();
ParseContext.Document doc = docMapper.parse("person", "1", json).rootDoc();
ParseContext.Document doc = docMapper.parse("person", "person", "1", json).rootDoc();
terminal.println("## Extracted text");
terminal.println("--------------------- BEGIN -----------------------");

4
src/test/java/org/elasticsearch/index/mapper/attachment/test/unit/AttachmentUnitTestCase.java
View File

@ -22,13 +22,13 @@ package org.elasticsearch.index.mapper.attachment.test.unit;
import org.elasticsearch.Version;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.test.ElasticsearchTestCase;
import org.elasticsearch.test.ESTestCase;
import org.junit.Before;
import org.junit.BeforeClass;
import static org.elasticsearch.index.mapper.attachment.test.MapperTestUtils.assumeCorrectLocale;
public class AttachmentUnitTestCase extends ElasticsearchTestCase {
public class AttachmentUnitTestCase extends ESTestCase {
/**
* We can have issues with some JVMs and Locale
* See https://github.com/elasticsearch/elasticsearch-mapper-attachments/issues/105