honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #17764 from rjernst/parse_doc_without_threadlocal 

Internal: Remove threadlocal from document parser
This commit is contained in:
Ryan Ernst 2016-04-14 15:17:45 -07:00
parent 6d8ab56350 c6dd17a2c6
commit 24130e1f30
6 changed files with 50 additions and 105 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java
View File

@ -127,11 +127,10 @@ public class MetaDataIndexUpgradeService extends AbstractComponent {
SimilarityService similarityService = new SimilarityService(indexSettings, Collections.emptyMap());
try (AnalysisService analysisService = new FakeAnalysisService(indexSettings)) {
try (MapperService mapperService = new MapperService(indexSettings, analysisService, similarityService, mapperRegistry, () -> null)) {
for (ObjectCursor<MappingMetaData> cursor : indexMetaData.getMappings().values()) {
MappingMetaData mappingMetaData = cursor.value;
mapperService.merge(mappingMetaData.type(), mappingMetaData.source(), MapperService.MergeReason.MAPPING_RECOVERY, false);
}
MapperService mapperService = new MapperService(indexSettings, analysisService, similarityService, mapperRegistry, () -> null);
for (ObjectCursor<MappingMetaData> cursor : indexMetaData.getMappings().values()) {
MappingMetaData mappingMetaData = cursor.value;
mapperService.merge(mappingMetaData.type(), mappingMetaData.source(), MapperService.MergeReason.MAPPING_RECOVERY, false);
}
}
} catch (Exception ex) {

2
core/src/main/java/org/elasticsearch/index/IndexService.java
View File

@ -239,7 +239,7 @@ public final class IndexService extends AbstractIndexComponent implements IndexC
}
}
} finally {
IOUtils.close(bitsetFilterCache, indexCache, mapperService, indexFieldData, analysisService, refreshTask, fsyncTask,
IOUtils.close(bitsetFilterCache, indexCache, indexFieldData, analysisService, refreshTask, fsyncTask,
cache().getPercolatorQueryCache());
}
}

4
core/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java
View File

@ -353,10 +353,6 @@ public class DocumentMapper implements ToXContent {
return new DocumentMapper(mapperService, updated);
}
public void close() {
documentParser.close();
}
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
return mapping.toXContent(builder, params);

24
core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java
View File

@ -19,7 +19,6 @@
package org.elasticsearch.index.mapper;
import java.io.Closeable;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
@ -28,7 +27,6 @@ import java.util.List;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.util.CloseableThreadLocal;
import org.elasticsearch.Version;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.joda.FormatDateTimeFormatter;
@ -38,12 +36,12 @@ import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.mapper.core.BinaryFieldMapper;
import org.elasticsearch.index.mapper.core.BooleanFieldMapper;
import org.elasticsearch.index.mapper.core.DateFieldMapper;
import org.elasticsearch.index.mapper.core.KeywordFieldMapper;
import org.elasticsearch.index.mapper.core.KeywordFieldMapper.KeywordFieldType;
import org.elasticsearch.index.mapper.core.LegacyDateFieldMapper;
import org.elasticsearch.index.mapper.core.LegacyDoubleFieldMapper;
import org.elasticsearch.index.mapper.core.LegacyFloatFieldMapper;
import org.elasticsearch.index.mapper.core.LegacyIntegerFieldMapper;
import org.elasticsearch.index.mapper.core.KeywordFieldMapper;
import org.elasticsearch.index.mapper.core.KeywordFieldMapper.KeywordFieldType;
import org.elasticsearch.index.mapper.core.LegacyLongFieldMapper;
import org.elasticsearch.index.mapper.core.NumberFieldMapper;
import org.elasticsearch.index.mapper.core.StringFieldMapper;
@ -57,14 +55,7 @@ import org.elasticsearch.index.mapper.object.ObjectMapper;
import org.elasticsearch.index.mapper.object.RootObjectMapper;
/** A parser for documents, given mappings from a DocumentMapper */
final class DocumentParser implements Closeable {
private CloseableThreadLocal<ParseContext.InternalParseContext> cache = new CloseableThreadLocal<ParseContext.InternalParseContext>() {
@Override
protected ParseContext.InternalParseContext initialValue() {
return new ParseContext.InternalParseContext(indexSettings.getSettings(), docMapperParser, docMapper, new ContentPath(0));
}
};
final class DocumentParser {
private final IndexSettings indexSettings;
private final DocumentMapperParser docMapperParser;
@ -81,7 +72,7 @@ final class DocumentParser implements Closeable {
source.type(docMapper.type());
final Mapping mapping = docMapper.mapping();
final ParseContext.InternalParseContext context = cache.get();
final ParseContext.InternalParseContext context = new ParseContext.InternalParseContext(indexSettings.getSettings(), docMapperParser, docMapper, new ContentPath(0));
XContentParser parser = null;
try {
parser = parser(source);
@ -101,8 +92,6 @@ final class DocumentParser implements Closeable {
reverseOrder(context);
ParsedDocument doc = parsedDocument(source, context, createDynamicUpdate(mapping, docMapper, context.getDynamicMappers()));
// reset the context to free up memory
context.reset(null, null, null);
return doc;
}
@ -932,9 +921,4 @@ final class DocumentParser implements Closeable {
private static ObjectMapper.Dynamic dynamicOrDefault(ObjectMapper.Dynamic dynamic) {
return dynamic == null ? ObjectMapper.Dynamic.TRUE : dynamic;
}
@Override
public void close() {
cache.close();
}
}

9
core/src/main/java/org/elasticsearch/index/mapper/MapperService.java
View File

@ -63,7 +63,7 @@ import static org.elasticsearch.common.collect.MapBuilder.newMapBuilder;
/**
*
*/
public class MapperService extends AbstractIndexComponent implements Closeable {
public class MapperService extends AbstractIndexComponent {
/**
* The reason why a mapping is being merged.
@ -166,13 +166,6 @@ public class MapperService extends AbstractIndexComponent implements Closeable {
}
}
@Override
public void close() {
for (DocumentMapper documentMapper : mappers.values()) {
documentMapper.close();
}
}
public boolean hasNested() {
return this.hasNested;
}

107
core/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java
View File

@ -19,11 +19,16 @@
package org.elasticsearch.index.mapper;
import org.elasticsearch.ExceptionsHelper;
import org.elasticsearch.Version;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.common.compress.CompressedXContent;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.concurrent.ExecutionException;
import java.util.function.Function;
import org.elasticsearch.ExceptionsHelper;
import org.elasticsearch.common.compress.CompressedXContent;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.IndexService;
@ -31,40 +36,23 @@ import org.elasticsearch.index.mapper.MapperService.MergeReason;
import org.elasticsearch.index.mapper.core.KeywordFieldMapper.KeywordFieldType;
import org.elasticsearch.index.mapper.core.NumberFieldMapper.NumberFieldType;
import org.elasticsearch.test.ESSingleNodeTestCase;
import org.junit.Rule;
import org.junit.rules.ExpectedException;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.function.Function;
import java.util.concurrent.ExecutionException;
import static org.hamcrest.CoreMatchers.containsString;
import static org.hamcrest.Matchers.hasToString;
import static org.hamcrest.Matchers.instanceOf;
public class MapperServiceTests extends ESSingleNodeTestCase {
@Rule
public ExpectedException expectedException = ExpectedException.none();
public void testTypeNameStartsWithIllegalDot() {
expectedException.expect(MapperParsingException.class);
expectedException.expect(hasToString(containsString("mapping type name [.test-type] must not start with a '.'")));
String index = "test-index";
String type = ".test-type";
String field = "field";
client()
.admin()
.indices()
.prepareCreate(index)
.addMapping(type, field, "type=text")
.execute()
.actionGet();
MapperParsingException e = expectThrows(MapperParsingException.class, () -> {
client().admin().indices().prepareCreate(index)
.addMapping(type, field, "type=text")
.execute().actionGet();
});
assertTrue(e.getMessage(), e.getMessage().contains("mapping type name [.test-type] must not start with a '.'"));
}
public void testTypeNameTooLong() {
@ -72,15 +60,12 @@ public class MapperServiceTests extends ESSingleNodeTestCase {
String field = "field";
String type = new String(new char[256]).replace("\0", "a");
expectedException.expect(MapperParsingException.class);
expectedException.expect(hasToString(containsString("mapping type name [" + type + "] is too long; limit is length 255 but was [256]")));
client()
.admin()
.indices()
.prepareCreate(index)
.addMapping(type, field, "type=text")
.execute()
.actionGet();
MapperParsingException e = expectThrows(MapperParsingException.class, () -> {
client().admin().indices().prepareCreate(index)
.addMapping(type, field, "type=text")
.execute().actionGet();
});
assertTrue(e.getMessage(), e.getMessage().contains("mapping type name [" + type + "] is too long; limit is length 255 but was [256]"));
}
public void testTypes() throws Exception {
@ -103,36 +88,26 @@ public class MapperServiceTests extends ESSingleNodeTestCase {
public void testIndexIntoDefaultMapping() throws Throwable {
// 1. test implicit index creation
try {
client().prepareIndex("index1", MapperService.DEFAULT_MAPPING, "1").setSource("{").execute().get();
fail();
} catch (Throwable t) {
if (t instanceof ExecutionException) {
t = t.getCause();
}
final Throwable throwable = ExceptionsHelper.unwrapCause(t);
if (throwable instanceof IllegalArgumentException) {
assertEquals("It is forbidden to index into the default mapping [_default_]", throwable.getMessage());
} else {
throw t;
}
ExecutionException e = expectThrows(ExecutionException.class, () -> {
client().prepareIndex("index1", MapperService.DEFAULT_MAPPING, "1").setSource("{}").execute().get();
});
Throwable throwable = ExceptionsHelper.unwrapCause(e.getCause());
if (throwable instanceof IllegalArgumentException) {
assertEquals("It is forbidden to index into the default mapping [_default_]", throwable.getMessage());
} else {
throw e;
}
// 2. already existing index
IndexService indexService = createIndex("index2");
try {
client().prepareIndex("index2", MapperService.DEFAULT_MAPPING, "2").setSource().execute().get();
fail();
} catch (Throwable t) {
if (t instanceof ExecutionException) {
t = t.getCause();
}
final Throwable throwable = ExceptionsHelper.unwrapCause(t);
if (throwable instanceof IllegalArgumentException) {
assertEquals("It is forbidden to index into the default mapping [_default_]", throwable.getMessage());
} else {
throw t;
}
expectThrows(ExecutionException.class, () -> {
client().prepareIndex("index1", MapperService.DEFAULT_MAPPING, "2").setSource().execute().get();
});
throwable = ExceptionsHelper.unwrapCause(e.getCause());
if (throwable instanceof IllegalArgumentException) {
assertEquals("It is forbidden to index into the default mapping [_default_]", throwable.getMessage());
} else {
throw e;
}
assertFalse(indexService.mapperService().hasMapping(MapperService.DEFAULT_MAPPING));
}
@ -149,13 +124,11 @@ public class MapperServiceTests extends ESSingleNodeTestCase {
};
createIndex("test1").mapperService().merge("type", new CompressedXContent(mapping.apply("type")), MergeReason.MAPPING_UPDATE, false);
//set total number of fields to 1 to trigger an exception
try {
IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> {
createIndex("test2", Settings.builder().put(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(), 1).build())
.mapperService().merge("type", new CompressedXContent(mapping.apply("type")), MergeReason.MAPPING_UPDATE, false);
fail("Expected IllegalArgumentException");
} catch (IllegalArgumentException e) {
assertThat(e.getMessage(), containsString("Limit of total fields [1] in index [test2] has been exceeded"));
}
});
assertTrue(e.getMessage(), e.getMessage().contains("Limit of total fields [1] in index [test2] has been exceeded"));
}
public void testMappingDepthExceedsLimit() throws Throwable {