Make field data changes immediately taken into account and add the ability to disallow field data loading.
This commit changes field data configuration updates so that they are immediately taken into account for loading new segments. The way it works is that field data configuration is now cached separately from the field data cache, meaning that it is now possible to clear the field data configuration from IndexFieldDataService while the cache will stay around. On the next time that Elasticsearch will reload field data configuration, it will check if there is already a cache entry, and reuse it if it exists. To disable field data loading, all that is required is to change the field data format to "none" (supported by all field data types) using the update mapping API. Elasticsearch will then refuse to load field data on any new segment, but field data which has been loaded on the previous segments will remain available. So you need to clear the field data cache in order to reclaim memory (otherwise memory will be reclaimed slower, as segments get merged). Close #4430 Close #4431
This commit is contained in:
Adrien Grand
parent
8d321530de
commit
4e7ce4ee02
|
|
@ -26,6 +26,8 @@ example, can be set to `5m` for a 5 minute expiry.
|
|||
|
||||
=== Field data formats
|
||||
|
||||
The field data format controls how field data should be stored.
|
||||
|
||||
Depending on the field type, there might be several field data types
|
||||
available. In particular, string and numeric types support the `doc_values`
|
||||
format which allows for computing the field data data-structures at indexing
|
||||
|
|
@ -33,6 +35,9 @@ time and storing them on disk. Although it will make the index larger and may
|
|||
be slightly slower, this implementation will be more near-realtime-friendly
|
||||
and will require much less memory from the JVM than other implementations.
|
||||
|
||||
Here is an example of how to configure the `tag` field to use the `fst` field
|
||||
data format.
|
||||
|
||||
[source,js]
|
||||
--------------------------------------------------
|
||||
{
|
||||
|
|
@ -45,6 +50,25 @@ and will require much less memory from the JVM than other implementations.
|
|||
}
|
||||
--------------------------------------------------
|
||||
|
||||
It is possible to change the field data format (and the field data settings
|
||||
in general) on a live index by using the update mapping API. When doing so,
|
||||
field data which had already been loaded for existing segments will remain
|
||||
alive while new segments will use the new field data configuration. Thanks to
|
||||
the background merging process, all segments will eventually use the new
|
||||
field data format.
|
||||
|
||||
[float]
|
||||
==== Disallowing field data loading
|
||||
|
||||
Field data can take a lot of RAM so it makes sense to disable field data
|
||||
loading on the fields that don't need field data, for example those that are
|
||||
used for full-text search only. In order to disable field data loading, just
|
||||
change the field data type to `disabled`. Request that will try to load field
|
||||
data on any field which is configured with this format will then return an
|
||||
error.
|
||||
|
||||
The `disabled` format is supported by all field types.
|
||||
|
||||
[float]
|
||||
==== String field data types
|
||||
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@
|
|||
package org.elasticsearch.index.fielddata;
|
||||
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.collect.Maps;
|
||||
import org.apache.lucene.index.IndexReader;
|
||||
import org.elasticsearch.ElasticSearchIllegalArgumentException;
|
||||
import org.elasticsearch.common.collect.MapBuilder;
|
||||
|
|
@ -36,12 +37,19 @@ import org.elasticsearch.index.service.IndexService;
|
|||
import org.elasticsearch.index.settings.IndexSettings;
|
||||
import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache;
|
||||
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ConcurrentMap;
|
||||
|
||||
/**
|
||||
*/
|
||||
public class IndexFieldDataService extends AbstractIndexComponent {
|
||||
|
||||
private static final String DISABLED_FORMAT = "disabled";
|
||||
private static final String DOC_VALUES_FORMAT = "doc_values";
|
||||
private static final String ARRAY_FORMAT = "array";
|
||||
private static final String PAGED_BYTES_FORMAT = "paged_bytes";
|
||||
private static final String FST_FORMAT = "fst";
|
||||
|
||||
private final static ImmutableMap<String, IndexFieldData.Builder> buildersByType;
|
||||
private final static ImmutableMap<String, IndexFieldData.Builder> docValuesBuildersByType;
|
||||
private final static ImmutableMap<Tuple<String, String>, IndexFieldData.Builder> buildersByTypeAndFormat;
|
||||
|
|
@ -69,30 +77,47 @@ public class IndexFieldDataService extends AbstractIndexComponent {
|
|||
.immutableMap();
|
||||
|
||||
buildersByTypeAndFormat = MapBuilder.<Tuple<String, String>, IndexFieldData.Builder>newMapBuilder()
|
||||
.put(Tuple.tuple("string", "paged_bytes"), new PagedBytesIndexFieldData.Builder())
|
||||
.put(Tuple.tuple("string", "fst"), new FSTBytesIndexFieldData.Builder())
|
||||
.put(Tuple.tuple("string", "doc_values"), new DocValuesIndexFieldData.Builder())
|
||||
.put(Tuple.tuple("float", "array"), new FloatArrayIndexFieldData.Builder())
|
||||
.put(Tuple.tuple("float", "doc_values"), new DocValuesIndexFieldData.Builder().numericType(IndexNumericFieldData.NumericType.FLOAT))
|
||||
.put(Tuple.tuple("double", "array"), new DoubleArrayIndexFieldData.Builder())
|
||||
.put(Tuple.tuple("double", "doc_values"), new DocValuesIndexFieldData.Builder().numericType(IndexNumericFieldData.NumericType.DOUBLE))
|
||||
.put(Tuple.tuple("byte", "array"), new PackedArrayIndexFieldData.Builder().setNumericType(IndexNumericFieldData.NumericType.BYTE))
|
||||
.put(Tuple.tuple("byte", "doc_values"), new DocValuesIndexFieldData.Builder().numericType(IndexNumericFieldData.NumericType.BYTE))
|
||||
.put(Tuple.tuple("short", "array"), new PackedArrayIndexFieldData.Builder().setNumericType(IndexNumericFieldData.NumericType.SHORT))
|
||||
.put(Tuple.tuple("short", "doc_values"), new DocValuesIndexFieldData.Builder().numericType(IndexNumericFieldData.NumericType.SHORT))
|
||||
.put(Tuple.tuple("int", "array"), new PackedArrayIndexFieldData.Builder().setNumericType(IndexNumericFieldData.NumericType.INT))
|
||||
.put(Tuple.tuple("int", "doc_values"), new DocValuesIndexFieldData.Builder().numericType(IndexNumericFieldData.NumericType.INT))
|
||||
.put(Tuple.tuple("long", "array"), new PackedArrayIndexFieldData.Builder().setNumericType(IndexNumericFieldData.NumericType.LONG))
|
||||
.put(Tuple.tuple("long", "doc_values"), new DocValuesIndexFieldData.Builder().numericType(IndexNumericFieldData.NumericType.LONG))
|
||||
.put(Tuple.tuple("geo_point", "array"), new GeoPointDoubleArrayIndexFieldData.Builder())
|
||||
.put(Tuple.tuple("string", PAGED_BYTES_FORMAT), new PagedBytesIndexFieldData.Builder())
|
||||
.put(Tuple.tuple("string", FST_FORMAT), new FSTBytesIndexFieldData.Builder())
|
||||
.put(Tuple.tuple("string", DOC_VALUES_FORMAT), new DocValuesIndexFieldData.Builder())
|
||||
.put(Tuple.tuple("string", DISABLED_FORMAT), new DisabledIndexFieldData.Builder())
|
||||
|
||||
.put(Tuple.tuple("float", ARRAY_FORMAT), new FloatArrayIndexFieldData.Builder())
|
||||
.put(Tuple.tuple("float", DOC_VALUES_FORMAT), new DocValuesIndexFieldData.Builder().numericType(IndexNumericFieldData.NumericType.FLOAT))
|
||||
.put(Tuple.tuple("float", DISABLED_FORMAT), new DisabledIndexFieldData.Builder())
|
||||
|
||||
.put(Tuple.tuple("double", ARRAY_FORMAT), new DoubleArrayIndexFieldData.Builder())
|
||||
.put(Tuple.tuple("double", DOC_VALUES_FORMAT), new DocValuesIndexFieldData.Builder().numericType(IndexNumericFieldData.NumericType.DOUBLE))
|
||||
.put(Tuple.tuple("double", DISABLED_FORMAT), new DisabledIndexFieldData.Builder())
|
||||
|
||||
.put(Tuple.tuple("byte", ARRAY_FORMAT), new PackedArrayIndexFieldData.Builder().setNumericType(IndexNumericFieldData.NumericType.BYTE))
|
||||
.put(Tuple.tuple("byte", DOC_VALUES_FORMAT), new DocValuesIndexFieldData.Builder().numericType(IndexNumericFieldData.NumericType.BYTE))
|
||||
.put(Tuple.tuple("byte", DISABLED_FORMAT), new DisabledIndexFieldData.Builder())
|
||||
|
||||
.put(Tuple.tuple("short", ARRAY_FORMAT), new PackedArrayIndexFieldData.Builder().setNumericType(IndexNumericFieldData.NumericType.SHORT))
|
||||
.put(Tuple.tuple("short", DOC_VALUES_FORMAT), new DocValuesIndexFieldData.Builder().numericType(IndexNumericFieldData.NumericType.SHORT))
|
||||
.put(Tuple.tuple("short", DISABLED_FORMAT), new DisabledIndexFieldData.Builder())
|
||||
|
||||
.put(Tuple.tuple("int", ARRAY_FORMAT), new PackedArrayIndexFieldData.Builder().setNumericType(IndexNumericFieldData.NumericType.INT))
|
||||
.put(Tuple.tuple("int", DOC_VALUES_FORMAT), new DocValuesIndexFieldData.Builder().numericType(IndexNumericFieldData.NumericType.INT))
|
||||
.put(Tuple.tuple("int", DISABLED_FORMAT), new DisabledIndexFieldData.Builder())
|
||||
|
||||
.put(Tuple.tuple("long", ARRAY_FORMAT), new PackedArrayIndexFieldData.Builder().setNumericType(IndexNumericFieldData.NumericType.LONG))
|
||||
.put(Tuple.tuple("long", DOC_VALUES_FORMAT), new DocValuesIndexFieldData.Builder().numericType(IndexNumericFieldData.NumericType.LONG))
|
||||
.put(Tuple.tuple("long", DISABLED_FORMAT), new DisabledIndexFieldData.Builder())
|
||||
|
||||
.put(Tuple.tuple("geo_point", ARRAY_FORMAT), new GeoPointDoubleArrayIndexFieldData.Builder())
|
||||
.put(Tuple.tuple("geo_point", DISABLED_FORMAT), new DisabledIndexFieldData.Builder())
|
||||
.immutableMap();
|
||||
}
|
||||
|
||||
private final IndicesFieldDataCache indicesFieldDataCache;
|
||||
private final ConcurrentMap<String, IndexFieldData<?>> loadedFieldData = ConcurrentCollections.newConcurrentMap();
|
||||
private final Map<String, IndexFieldDataCache> fieldDataCaches = Maps.newHashMap(); // no need for concurrency support, always used under lock
|
||||
|
||||
IndexService indexService;
|
||||
|
||||
// public for testing
|
||||
public IndexFieldDataService(Index index) {
|
||||
this(index, ImmutableSettings.Builder.EMPTY_SETTINGS, new IndicesFieldDataCache(ImmutableSettings.Builder.EMPTY_SETTINGS));
|
||||
}
|
||||
|
|
@ -114,6 +139,10 @@ public class IndexFieldDataService extends AbstractIndexComponent {
|
|||
fieldData.clear();
|
||||
}
|
||||
loadedFieldData.clear();
|
||||
for (IndexFieldDataCache cache : fieldDataCaches.values()) {
|
||||
cache.clear();
|
||||
}
|
||||
fieldDataCaches.clear();
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -123,12 +152,29 @@ public class IndexFieldDataService extends AbstractIndexComponent {
|
|||
if (fieldData != null) {
|
||||
fieldData.clear();
|
||||
}
|
||||
IndexFieldDataCache cache = fieldDataCaches.remove(fieldName);
|
||||
if (cache != null) {
|
||||
cache.clear();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void clear(IndexReader reader) {
|
||||
for (IndexFieldData<?> indexFieldData : loadedFieldData.values()) {
|
||||
indexFieldData.clear(reader);
|
||||
synchronized (loadedFieldData) {
|
||||
for (IndexFieldData<?> indexFieldData : loadedFieldData.values()) {
|
||||
indexFieldData.clear(reader);
|
||||
}
|
||||
for (IndexFieldDataCache cache : fieldDataCaches.values()) {
|
||||
cache.clear(reader);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void onMappingUpdate() {
|
||||
// synchronize to make sure to not miss field data instances that are being loaded
|
||||
synchronized (loadedFieldData) {
|
||||
// important: do not clear fieldDataCaches: the cache may be reused
|
||||
loadedFieldData.clear();
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -163,18 +209,21 @@ public class IndexFieldDataService extends AbstractIndexComponent {
|
|||
throw new ElasticSearchIllegalArgumentException("failed to find field data builder for field " + fieldNames.fullName() + ", and type " + type.getType());
|
||||
}
|
||||
|
||||
IndexFieldDataCache cache;
|
||||
// we default to node level cache, which in turn defaults to be unbounded
|
||||
// this means changing the node level settings is simple, just set the bounds there
|
||||
String cacheType = type.getSettings().get("cache", indexSettings.get("index.fielddata.cache", "node"));
|
||||
if ("resident".equals(cacheType)) {
|
||||
cache = new IndexFieldDataCache.Resident(indexService, fieldNames, type);
|
||||
} else if ("soft".equals(cacheType)) {
|
||||
cache = new IndexFieldDataCache.Soft(indexService, fieldNames, type);
|
||||
} else if ("node".equals(cacheType)) {
|
||||
cache = indicesFieldDataCache.buildIndexFieldDataCache(indexService, index, fieldNames, type);
|
||||
} else {
|
||||
throw new ElasticSearchIllegalArgumentException("cache type not supported [" + cacheType + "] for field [" + fieldNames.fullName() + "]");
|
||||
IndexFieldDataCache cache = fieldDataCaches.get(fieldNames.indexName());
|
||||
if (cache == null) {
|
||||
// we default to node level cache, which in turn defaults to be unbounded
|
||||
// this means changing the node level settings is simple, just set the bounds there
|
||||
String cacheType = type.getSettings().get("cache", indexSettings.get("index.fielddata.cache", "node"));
|
||||
if ("resident".equals(cacheType)) {
|
||||
cache = new IndexFieldDataCache.Resident(indexService, fieldNames, type);
|
||||
} else if ("soft".equals(cacheType)) {
|
||||
cache = new IndexFieldDataCache.Soft(indexService, fieldNames, type);
|
||||
} else if ("node".equals(cacheType)) {
|
||||
cache = indicesFieldDataCache.buildIndexFieldDataCache(indexService, index, fieldNames, type);
|
||||
} else {
|
||||
throw new ElasticSearchIllegalArgumentException("cache type not supported [" + cacheType + "] for field [" + fieldNames.fullName() + "]");
|
||||
}
|
||||
fieldDataCaches.put(fieldNames.indexName(), cache);
|
||||
}
|
||||
|
||||
fieldData = builder.build(index, indexSettings, fieldNames, type, cache);
|
||||
|
|
@ -184,4 +233,5 @@ public class IndexFieldDataService extends AbstractIndexComponent {
|
|||
}
|
||||
return (IFD) fieldData;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,68 @@
|
|||
/*
|
||||
* Licensed to ElasticSearch and Shay Banon under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. ElasticSearch licenses this
|
||||
* file to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.index.fielddata.plain;
|
||||
|
||||
import org.apache.lucene.index.AtomicReaderContext;
|
||||
import org.elasticsearch.ElasticSearchIllegalStateException;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.index.Index;
|
||||
import org.elasticsearch.index.fielddata.*;
|
||||
import org.elasticsearch.index.fielddata.fieldcomparator.SortMode;
|
||||
import org.elasticsearch.index.mapper.FieldMapper;
|
||||
import org.elasticsearch.index.mapper.FieldMapper.Names;
|
||||
import org.elasticsearch.index.settings.IndexSettings;
|
||||
|
||||
/**
|
||||
* A field data implementation that forbids loading and will throw an {@link ElasticSearchIllegalStateException} if you try to load
|
||||
* {@link AtomicFieldData} instances.
|
||||
*/
|
||||
public final class DisabledIndexFieldData extends AbstractIndexFieldData<AtomicFieldData<?>> {
|
||||
|
||||
public static class Builder implements IndexFieldData.Builder {
|
||||
@Override
|
||||
public IndexFieldData<AtomicFieldData<?>> build(Index index, @IndexSettings Settings indexSettings, FieldMapper.Names fieldNames, FieldDataType type, IndexFieldDataCache cache) {
|
||||
return new DisabledIndexFieldData(index, indexSettings, fieldNames, type, cache);
|
||||
}
|
||||
}
|
||||
|
||||
public DisabledIndexFieldData(Index index, Settings indexSettings, Names fieldNames, FieldDataType fieldDataType, IndexFieldDataCache cache) {
|
||||
super(index, indexSettings, fieldNames, fieldDataType, cache);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean valuesOrdered() {
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public AtomicFieldData<?> loadDirect(AtomicReaderContext context) throws Exception {
|
||||
throw fail();
|
||||
}
|
||||
|
||||
@Override
|
||||
public IndexFieldData.XFieldComparatorSource comparatorSource(Object missingValue, SortMode sortMode) {
|
||||
throw fail();
|
||||
}
|
||||
|
||||
private ElasticSearchIllegalStateException fail() {
|
||||
return new ElasticSearchIllegalStateException("Field data loading is forbidden on " + getFieldNames().name());
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -48,6 +48,7 @@ import org.elasticsearch.index.Index;
|
|||
import org.elasticsearch.index.analysis.AnalysisService;
|
||||
import org.elasticsearch.index.codec.docvaluesformat.DocValuesFormatService;
|
||||
import org.elasticsearch.index.codec.postingsformat.PostingsFormatService;
|
||||
import org.elasticsearch.index.fielddata.IndexFieldDataService;
|
||||
import org.elasticsearch.index.mapper.internal.TypeFieldMapper;
|
||||
import org.elasticsearch.index.mapper.object.ObjectMapper;
|
||||
import org.elasticsearch.index.search.nested.NonNestedDocsFilter;
|
||||
|
|
@ -76,6 +77,7 @@ public class MapperService extends AbstractIndexComponent implements Iterable<Do
|
|||
public static final String DEFAULT_MAPPING = "_default_";
|
||||
|
||||
private final AnalysisService analysisService;
|
||||
private final IndexFieldDataService fieldDataService;
|
||||
|
||||
/**
|
||||
* Will create types automatically if they do not exists in the mapping definition yet
|
||||
|
|
@ -105,10 +107,11 @@ public class MapperService extends AbstractIndexComponent implements Iterable<Do
|
|||
private final List<DocumentTypeListener> typeListeners = new CopyOnWriteArrayList<DocumentTypeListener>();
|
||||
|
||||
@Inject
|
||||
public MapperService(Index index, @IndexSettings Settings indexSettings, Environment environment, AnalysisService analysisService,
|
||||
public MapperService(Index index, @IndexSettings Settings indexSettings, Environment environment, AnalysisService analysisService, IndexFieldDataService fieldDataService,
|
||||
PostingsFormatService postingsFormatService, DocValuesFormatService docValuesFormatService, SimilarityLookupService similarityLookupService) {
|
||||
super(index, indexSettings);
|
||||
this.analysisService = analysisService;
|
||||
this.fieldDataService = fieldDataService;
|
||||
this.documentParser = new DocumentMapperParser(index, indexSettings, analysisService, postingsFormatService, docValuesFormatService, similarityLookupService);
|
||||
this.searchAnalyzer = new SmartIndexNameSearchAnalyzer(analysisService.defaultSearchAnalyzer());
|
||||
this.searchQuoteAnalyzer = new SmartIndexNameSearchQuoteAnalyzer(analysisService.defaultSearchQuoteAnalyzer());
|
||||
|
|
@ -278,6 +281,7 @@ public class MapperService extends AbstractIndexComponent implements Iterable<Do
|
|||
logger.debug("merging mapping for type [{}] resulted in conflicts: [{}]", mapper.type(), Arrays.toString(result.conflicts()));
|
||||
}
|
||||
}
|
||||
fieldDataService.onMappingUpdate();
|
||||
return oldMapper;
|
||||
} else {
|
||||
FieldMapperListener.Aggregator fieldMappersAgg = new FieldMapperListener.Aggregator();
|
||||
|
|
|
|||
|
|
@ -0,0 +1,93 @@
|
|||
/*
|
||||
* Licensed to ElasticSearch and Shay Banon under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. ElasticSearch licenses this
|
||||
* file to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.index.fielddata;
|
||||
|
||||
import org.elasticsearch.action.search.SearchPhaseExecutionException;
|
||||
import org.elasticsearch.action.search.SearchResponse;
|
||||
import org.elasticsearch.common.xcontent.XContentFactory;
|
||||
import org.elasticsearch.search.aggregations.AggregationBuilders;
|
||||
import org.elasticsearch.test.ElasticsearchIntegrationTest;
|
||||
|
||||
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
|
||||
|
||||
public class DisabledFieldDataFormatTests extends ElasticsearchIntegrationTest {
|
||||
|
||||
public void test() throws Exception {
|
||||
createIndex("test");
|
||||
ensureGreen();
|
||||
for (int i = 0; i < 10; ++i) {
|
||||
client().prepareIndex("test", "type", Integer.toString(i)).setSource("s", "value" + i).execute().actionGet();
|
||||
}
|
||||
|
||||
refresh();
|
||||
|
||||
// disable field data
|
||||
updateFormat("disabled");
|
||||
|
||||
SearchResponse resp = null;
|
||||
// try to run something that relies on field data and make sure that it fails
|
||||
try {
|
||||
resp = client().prepareSearch("test").addAggregation(AggregationBuilders.terms("t").field("s")).execute().actionGet();
|
||||
assertTrue(resp.toString(), resp.getFailedShards() > 0);
|
||||
} catch (SearchPhaseExecutionException e) {
|
||||
// expected
|
||||
}
|
||||
|
||||
// enable it again
|
||||
updateFormat("paged_bytes");
|
||||
|
||||
// try to run something that relies on field data and make sure that it works
|
||||
resp = client().prepareSearch("test").addAggregation(AggregationBuilders.terms("t").field("s")).execute().actionGet();
|
||||
assertNoFailures(resp);
|
||||
|
||||
// disable it again
|
||||
updateFormat("disabled");
|
||||
|
||||
// this time, it should work because segments are already loaded
|
||||
resp = client().prepareSearch("test").addAggregation(AggregationBuilders.terms("t").field("s")).execute().actionGet();
|
||||
assertNoFailures(resp);
|
||||
|
||||
// but add more docs and the new segment won't be loaded
|
||||
client().prepareIndex("test", "type", "-1").setSource("s", "value").execute().actionGet();
|
||||
refresh();
|
||||
try {
|
||||
resp = client().prepareSearch("test").addAggregation(AggregationBuilders.terms("t").field("s")).execute().actionGet();
|
||||
assertTrue(resp.toString(), resp.getFailedShards() > 0);
|
||||
} catch (SearchPhaseExecutionException e) {
|
||||
// expected
|
||||
}
|
||||
}
|
||||
|
||||
private void updateFormat(String format) throws Exception {
|
||||
client().admin().indices().preparePutMapping("test").setType("type").setSource(
|
||||
XContentFactory.jsonBuilder().startObject().startObject("type")
|
||||
.startObject("properties")
|
||||
.startObject("s")
|
||||
.field("type", "string")
|
||||
.startObject("fielddata")
|
||||
.field("format", format)
|
||||
.endObject()
|
||||
.endObject()
|
||||
.endObject()
|
||||
.endObject()
|
||||
.endObject()).execute().actionGet();
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -19,6 +19,12 @@
|
|||
|
||||
package org.elasticsearch.index.fielddata;
|
||||
|
||||
import org.apache.lucene.analysis.core.KeywordAnalyzer;
|
||||
import org.apache.lucene.document.Document;
|
||||
import org.apache.lucene.document.Field.Store;
|
||||
import org.apache.lucene.document.StringField;
|
||||
import org.apache.lucene.index.*;
|
||||
import org.apache.lucene.store.RAMDirectory;
|
||||
import org.elasticsearch.common.settings.ImmutableSettings;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.index.Index;
|
||||
|
|
@ -26,10 +32,16 @@ import org.elasticsearch.index.fielddata.plain.*;
|
|||
import org.elasticsearch.index.mapper.ContentPath;
|
||||
import org.elasticsearch.index.mapper.FieldMapper;
|
||||
import org.elasticsearch.index.mapper.Mapper.BuilderContext;
|
||||
import org.elasticsearch.index.mapper.MapperBuilders;
|
||||
import org.elasticsearch.index.mapper.core.*;
|
||||
import org.elasticsearch.test.ElasticsearchTestCase;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.IdentityHashMap;
|
||||
import java.util.Set;
|
||||
|
||||
import static org.hamcrest.Matchers.instanceOf;
|
||||
|
||||
public class IndexFieldDataServiceTests extends ElasticsearchTestCase {
|
||||
|
||||
|
|
@ -88,7 +100,7 @@ public class IndexFieldDataServiceTests extends ElasticsearchTestCase {
|
|||
public void testByPassDocValues() {
|
||||
final IndexFieldDataService ifdService = new IndexFieldDataService(new Index("test"));
|
||||
final BuilderContext ctx = new BuilderContext(null, new ContentPath(1));
|
||||
final StringFieldMapper stringMapper = new StringFieldMapper.Builder("string").tokenized(false).fieldDataSettings(DOC_VALUES_SETTINGS).fieldDataSettings(ImmutableSettings.builder().put("format", "fst").build()).build(ctx);
|
||||
final StringFieldMapper stringMapper = MapperBuilders.stringField("string").tokenized(false).fieldDataSettings(DOC_VALUES_SETTINGS).fieldDataSettings(ImmutableSettings.builder().put("format", "fst").build()).build(ctx);
|
||||
ifdService.clear();
|
||||
IndexFieldData<?> fd = ifdService.getForField(stringMapper);
|
||||
assertTrue(fd instanceof FSTBytesIndexFieldData);
|
||||
|
|
@ -105,15 +117,53 @@ public class IndexFieldDataServiceTests extends ElasticsearchTestCase {
|
|||
assertTrue(fd instanceof PackedArrayIndexFieldData);
|
||||
}
|
||||
|
||||
final FloatFieldMapper floatMapper = new FloatFieldMapper.Builder("float").fieldDataSettings(DOC_VALUES_SETTINGS).fieldDataSettings(fdSettings).build(ctx);
|
||||
final FloatFieldMapper floatMapper = MapperBuilders.floatField("float").fieldDataSettings(DOC_VALUES_SETTINGS).fieldDataSettings(fdSettings).build(ctx);
|
||||
ifdService.clear();
|
||||
fd = ifdService.getForField(floatMapper);
|
||||
assertTrue(fd instanceof FloatArrayIndexFieldData);
|
||||
|
||||
final DoubleFieldMapper doubleMapper = new DoubleFieldMapper.Builder("double").fieldDataSettings(DOC_VALUES_SETTINGS).fieldDataSettings(fdSettings).build(ctx);
|
||||
final DoubleFieldMapper doubleMapper = MapperBuilders.doubleField("double").fieldDataSettings(DOC_VALUES_SETTINGS).fieldDataSettings(fdSettings).build(ctx);
|
||||
ifdService.clear();
|
||||
fd = ifdService.getForField(doubleMapper);
|
||||
assertTrue(fd instanceof DoubleArrayIndexFieldData);
|
||||
}
|
||||
|
||||
public void testChangeFieldDataFormat() throws Exception {
|
||||
final IndexFieldDataService ifdService = new IndexFieldDataService(new Index("test"));
|
||||
final BuilderContext ctx = new BuilderContext(null, new ContentPath(1));
|
||||
final StringFieldMapper mapper1 = MapperBuilders.stringField("s").tokenized(false).fieldDataSettings(ImmutableSettings.builder().put(FieldDataType.FORMAT_KEY, "paged_bytes").build()).build(ctx);
|
||||
final IndexWriter writer = new IndexWriter(new RAMDirectory(), new IndexWriterConfig(TEST_VERSION_CURRENT, new KeywordAnalyzer()));
|
||||
Document doc = new Document();
|
||||
doc.add(new StringField("s", "thisisastring", Store.NO));
|
||||
writer.addDocument(doc);
|
||||
final IndexReader reader1 = DirectoryReader.open(writer, true);
|
||||
IndexFieldData<?> ifd = ifdService.getForField(mapper1);
|
||||
assertThat(ifd, instanceOf(PagedBytesIndexFieldData.class));
|
||||
Set<AtomicReader> oldSegments = Collections.newSetFromMap(new IdentityHashMap<AtomicReader, Boolean>());
|
||||
for (AtomicReaderContext arc : reader1.leaves()) {
|
||||
oldSegments.add(arc.reader());
|
||||
AtomicFieldData<?> afd = ifd.load(arc);
|
||||
assertThat(afd, instanceOf(PagedBytesAtomicFieldData.class));
|
||||
}
|
||||
// write new segment
|
||||
writer.addDocument(doc);
|
||||
final IndexReader reader2 = DirectoryReader.open(writer, true);
|
||||
final StringFieldMapper mapper2 = MapperBuilders.stringField("s").tokenized(false).fieldDataSettings(ImmutableSettings.builder().put(FieldDataType.FORMAT_KEY, "fst").build()).build(ctx);
|
||||
ifdService.onMappingUpdate();
|
||||
ifd = ifdService.getForField(mapper2);
|
||||
assertThat(ifd, instanceOf(FSTBytesIndexFieldData.class));
|
||||
for (AtomicReaderContext arc : reader2.leaves()) {
|
||||
AtomicFieldData<?> afd = ifd.load(arc);
|
||||
if (oldSegments.contains(arc.reader())) {
|
||||
assertThat(afd, instanceOf(PagedBytesAtomicFieldData.class));
|
||||
} else {
|
||||
assertThat(afd, instanceOf(FSTBytesAtomicFieldData.class));
|
||||
}
|
||||
}
|
||||
reader1.close();
|
||||
reader2.close();
|
||||
writer.close();
|
||||
writer.getDirectory().close();
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -32,6 +32,7 @@ import org.elasticsearch.index.analysis.AnalysisModule;
|
|||
import org.elasticsearch.index.analysis.AnalysisService;
|
||||
import org.elasticsearch.index.codec.docvaluesformat.DocValuesFormatService;
|
||||
import org.elasticsearch.index.codec.postingsformat.PostingsFormatService;
|
||||
import org.elasticsearch.index.fielddata.IndexFieldDataService;
|
||||
import org.elasticsearch.index.settings.IndexSettingsModule;
|
||||
import org.elasticsearch.index.similarity.SimilarityLookupService;
|
||||
import org.elasticsearch.indices.analysis.IndicesAnalysisModule;
|
||||
|
|
@ -53,8 +54,12 @@ public class MapperTestUtils {
|
|||
}
|
||||
|
||||
public static MapperService newMapperService() {
|
||||
return new MapperService(new Index("test"), ImmutableSettings.Builder.EMPTY_SETTINGS, new Environment(), newAnalysisService(),
|
||||
new PostingsFormatService(new Index("test")), new DocValuesFormatService(new Index("test")), newSimilarityLookupService());
|
||||
return newMapperService(new Index("test"), ImmutableSettings.Builder.EMPTY_SETTINGS);
|
||||
}
|
||||
|
||||
public static MapperService newMapperService(Index index, Settings indexSettings) {
|
||||
return new MapperService(index, indexSettings, new Environment(), newAnalysisService(), new IndexFieldDataService(index),
|
||||
new PostingsFormatService(index), new DocValuesFormatService(index), newSimilarityLookupService());
|
||||
}
|
||||
|
||||
public static AnalysisService newAnalysisService() {
|
||||
|
|
|
|||
|
|
@ -36,15 +36,14 @@ import org.elasticsearch.common.lucene.search.XConstantScoreQuery;
|
|||
import org.elasticsearch.common.lucene.search.XFilteredQuery;
|
||||
import org.elasticsearch.common.settings.ImmutableSettings;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.env.Environment;
|
||||
import org.elasticsearch.index.Index;
|
||||
import org.elasticsearch.index.analysis.AnalysisService;
|
||||
import org.elasticsearch.index.cache.filter.weighted.WeightedFilterCache;
|
||||
import org.elasticsearch.index.cache.id.IdCache;
|
||||
import org.elasticsearch.index.cache.id.SimpleIdCacheTests;
|
||||
import org.elasticsearch.index.cache.id.simple.SimpleIdCache;
|
||||
import org.elasticsearch.index.engine.Engine;
|
||||
import org.elasticsearch.index.mapper.MapperService;
|
||||
import org.elasticsearch.index.mapper.MapperTestUtils;
|
||||
import org.elasticsearch.index.mapper.Uid;
|
||||
import org.elasticsearch.index.mapper.internal.ParentFieldMapper;
|
||||
import org.elasticsearch.index.mapper.internal.TypeFieldMapper;
|
||||
|
|
@ -328,9 +327,7 @@ public class ChildrenConstantScoreQueryTests extends ElasticsearchLuceneTestCase
|
|||
final IdCache idCache = new SimpleIdCache(index, ImmutableSettings.EMPTY);
|
||||
final CacheRecycler cacheRecycler = new CacheRecycler(ImmutableSettings.EMPTY);
|
||||
Settings settings = ImmutableSettings.EMPTY;
|
||||
MapperService mapperService = new MapperService(
|
||||
index, settings, new Environment(), new AnalysisService(index), null, null, null
|
||||
);
|
||||
MapperService mapperService = MapperTestUtils.newMapperService(index, settings);
|
||||
mapperService.merge(
|
||||
childType, new CompressedString(PutMappingRequest.buildFromSimplifiedDef(childType, "_parent", "type=" + parentType).string()), true
|
||||
);
|
||||
|
|
|
|||
Loading…
Reference in New Issue