honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Remove `omit_term_freq_and_positions` for new indices 

`omit_term_freq_and_positions` was deprecated in `0.20` and
is not documented anymore. We should reject indices that are
created with this option in the future.

Closes #4722
This commit is contained in:
Simon Willnauer 2014-01-14 13:39:52 +01:00
parent 0d8330b50a
commit da707b6f32
6 changed files with 51 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java
View File

@ -21,13 +21,14 @@ package org.elasticsearch.index.mapper;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Maps;
import org.elasticsearch.Version;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.collect.MapBuilder;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.compress.CompressedString;
import org.elasticsearch.common.geo.ShapesAvailability;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentHelper;
@ -64,16 +65,11 @@ public class DocumentMapperParser extends AbstractIndexComponent {
private final RootObjectMapper.TypeParser rootObjectTypeParser = new RootObjectMapper.TypeParser();
private final Object typeParsersMutex = new Object();
private final Version indexVersionCreated;
private volatile ImmutableMap<String, Mapper.TypeParser> typeParsers;
private volatile ImmutableMap<String, Mapper.TypeParser> rootTypeParsers;
public DocumentMapperParser(Index index, AnalysisService analysisService, PostingsFormatService postingsFormatService,
DocValuesFormatService docValuesFormatService, SimilarityLookupService similarityLookupService) {
this(index, ImmutableSettings.Builder.EMPTY_SETTINGS, analysisService, postingsFormatService, docValuesFormatService,
similarityLookupService);
}
public DocumentMapperParser(Index index, @IndexSettings Settings indexSettings, AnalysisService analysisService,
PostingsFormatService postingsFormatService, DocValuesFormatService docValuesFormatService,
SimilarityLookupService similarityLookupService) {
@ -123,6 +119,9 @@ public class DocumentMapperParser extends AbstractIndexComponent {
.put(VersionFieldMapper.NAME, new VersionFieldMapper.TypeParser())
.put(IdFieldMapper.NAME, new IdFieldMapper.TypeParser())
.immutableMap();
assert indexSettings.get(IndexMetaData.SETTING_UUID) == null // if the UUDI is there the index has actually been created otherwise this might be a test
|| indexSettings.getAsVersion(IndexMetaData.SETTING_VERSION_CREATED, null) != null : IndexMetaData.SETTING_VERSION_CREATED + " not set in IndexSettings";
indexVersionCreated = indexSettings.getAsVersion(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT);
}
public void putTypeParser(String type, Mapper.TypeParser typeParser) {
@ -142,7 +141,7 @@ public class DocumentMapperParser extends AbstractIndexComponent {
}
public Mapper.TypeParser.ParserContext parserContext() {
return new Mapper.TypeParser.ParserContext(postingsFormatService, docValuesFormatService, analysisService, similarityLookupService, typeParsers);
return new Mapper.TypeParser.ParserContext(postingsFormatService, docValuesFormatService, analysisService, similarityLookupService, typeParsers, indexVersionCreated);
}
public DocumentMapper parse(String source) throws MapperParsingException {
@ -199,8 +198,8 @@ public class DocumentMapperParser extends AbstractIndexComponent {
}
}
Mapper.TypeParser.ParserContext parserContext = new Mapper.TypeParser.ParserContext(postingsFormatService, docValuesFormatService, analysisService, similarityLookupService, typeParsers);
Mapper.TypeParser.ParserContext parserContext = parserContext();
DocumentMapper.Builder docBuilder = doc(index.name(), indexSettings, (RootObjectMapper.Builder) rootObjectTypeParser.parse(type, mapping, parserContext));
for (Map.Entry<String, Object> entry : mapping.entrySet()) {

9
src/main/java/org/elasticsearch/index/mapper/Mapper.java
View File

@ -99,14 +99,17 @@ public interface Mapper extends ToXContent {
private final ImmutableMap<String, TypeParser> typeParsers;
private final Version indexVersionCreated;
public ParserContext(PostingsFormatService postingsFormatService, DocValuesFormatService docValuesFormatService,
AnalysisService analysisService, SimilarityLookupService similarityLookupService,
ImmutableMap<String, TypeParser> typeParsers) {
ImmutableMap<String, TypeParser> typeParsers, Version indexVersionCreated) {
this.postingsFormatService = postingsFormatService;
this.docValuesFormatService = docValuesFormatService;
this.analysisService = analysisService;
this.similarityLookupService = similarityLookupService;
this.typeParsers = typeParsers;
this.indexVersionCreated = indexVersionCreated;
}
public AnalysisService analysisService() {
@ -128,6 +131,10 @@ public interface Mapper extends ToXContent {
public TypeParser typeParser(String type) {
return typeParsers.get(Strings.toUnderscoreCase(type));
}
public Version indexVersionCreated() {
return indexVersionCreated;
}
}
Mapper.Builder<?,?> parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException;

7
src/main/java/org/elasticsearch/index/mapper/core/TypeParsers.java
View File

@ -21,6 +21,7 @@ package org.elasticsearch.index.mapper.core;
import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.Version;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.joda.FormatDateTimeFormatter;
import org.elasticsearch.common.joda.Joda;
@ -198,8 +199,12 @@ public class TypeParsers {
}
}
} else if (propName.equals("omit_term_freq_and_positions")) {
final IndexOptions op = nodeBooleanValue(propNode) ? IndexOptions.DOCS_ONLY : IndexOptions.DOCS_AND_FREQS_AND_POSITIONS;
if (parserContext.indexVersionCreated().onOrAfter(Version.V_1_0_0)) {
throw new ElasticsearchParseException("'omit_term_freq_and_positions' is not supported anymore - use ['index_options' : '" + op.name() + "'] instead");
}
// deprecated option for BW compat
builder.indexOptions(nodeBooleanValue(propNode) ? IndexOptions.DOCS_ONLY : IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
builder.indexOptions(op);
} else if (propName.equals("index_options")) {
builder.indexOptions(nodeIndexOptionValue(propNode));
} else if (propName.equals("analyzer")) {

19
src/test/java/org/elasticsearch/count/query/SimpleQueryTests.java
View File

@ -130,25 +130,6 @@ public class SimpleQueryTests extends ElasticsearchIntegrationTest {
assertHitCount(countResponse, 3l);
}
@Test
public void testOmitTermFreqsAndPositions() throws Exception {
// backwards compat test!
assertAcked(prepareCreate("test")
.addMapping("type1", "field1", "type=string,omit_term_freq_and_positions=true")
.setSettings(SETTING_NUMBER_OF_SHARDS, 1));
indexRandom(true, client().prepareIndex("test", "type1", "1").setSource("field1", "quick brown fox", "field2", "quick brown fox"),
client().prepareIndex("test", "type1", "2").setSource("field1", "quick lazy huge brown fox", "field2", "quick lazy huge brown fox"));
CountResponse countResponse = client().prepareCount().setQuery(QueryBuilders.matchQuery("field2", "quick brown").type(Type.PHRASE).slop(0)).get();
assertHitCount(countResponse, 1l);
try {
client().prepareCount().setQuery(QueryBuilders.matchQuery("field1", "quick brown").type(Type.PHRASE).slop(0)).get();
} catch (SearchPhaseExecutionException e) {
assertTrue(e.getMessage().endsWith("IllegalStateException[field \"field1\" was indexed without position data; cannot run PhraseQuery (term=quick)]; }"));
}
}
@Test
public void queryStringAnalyzedWildcard() throws Exception {
assertAcked(prepareCreate("test").setSettings(SETTING_NUMBER_OF_SHARDS, 1));

2
src/test/java/org/elasticsearch/index/mapper/MapperTestUtils.java
View File

@ -45,7 +45,7 @@ import org.elasticsearch.indices.fielddata.breaker.DummyCircuitBreakerService;
public class MapperTestUtils {
public static DocumentMapperParser newParser() {
return new DocumentMapperParser(new Index("test"), newAnalysisService(), new PostingsFormatService(new Index("test")),
return new DocumentMapperParser(new Index("test"), ImmutableSettings.Builder.EMPTY_SETTINGS, newAnalysisService(), new PostingsFormatService(new Index("test")),
new DocValuesFormatService(new Index("test")), newSimilarityLookupService());
}

42
src/test/java/org/elasticsearch/search/query/SimpleQueryTests.java
View File

@ -21,13 +21,16 @@ package org.elasticsearch.search.query;
import org.apache.lucene.util.English;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.Version;
import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder;
import org.elasticsearch.action.index.IndexRequestBuilder;
import org.elasticsearch.action.search.SearchPhaseExecutionException;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchType;
import org.elasticsearch.action.search.ShardSearchFailure;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.mapper.MapperParsingException;
import org.elasticsearch.index.query.*;
import org.elasticsearch.index.query.CommonTermsQueryBuilder.Operator;
import org.elasticsearch.index.query.MatchQueryBuilder.Type;
@ -325,21 +328,32 @@ public class SimpleQueryTests extends ElasticsearchIntegrationTest {
@Test
public void testOmitTermFreqsAndPositions() throws Exception {
// backwards compat test!
assertAcked(client().admin().indices().prepareCreate("test")
.addMapping("type1", "field1", "type=string,omit_term_freq_and_positions=true")
.setSettings(SETTING_NUMBER_OF_SHARDS, 1));
Version version = Version.CURRENT;
int iters = atLeast(10);
for (int i = 0; i < iters; i++) {
try {
// backwards compat test!
assertAcked(client().admin().indices().prepareCreate("test")
.addMapping("type1", "field1", "type=string,omit_term_freq_and_positions=true")
.setSettings(SETTING_NUMBER_OF_SHARDS, 1, IndexMetaData.SETTING_VERSION_CREATED, version.id));
assertThat(version.onOrAfter(Version.V_1_0_0), equalTo(false));
indexRandom(true, client().prepareIndex("test", "type1", "1").setSource("field1", "quick brown fox", "field2", "quick brown fox"),
client().prepareIndex("test", "type1", "2").setSource("field1", "quick lazy huge brown fox", "field2", "quick lazy huge brown fox"));
indexRandom(true, client().prepareIndex("test", "type1", "1").setSource("field1", "quick brown fox", "field2", "quick brown fox"),
client().prepareIndex("test", "type1", "2").setSource("field1", "quick lazy huge brown fox", "field2", "quick lazy huge brown fox"));
SearchResponse searchResponse = client().prepareSearch().setQuery(matchQuery("field2", "quick brown").type(MatchQueryBuilder.Type.PHRASE).slop(0)).get();
assertHitCount(searchResponse, 1l);
try {
client().prepareSearch().setQuery(matchQuery("field1", "quick brown").type(MatchQueryBuilder.Type.PHRASE).slop(0)).get();
fail("SearchPhaseExecutionException should have been thrown");
} catch (SearchPhaseExecutionException e) {
assertTrue(e.getMessage().endsWith("IllegalStateException[field \"field1\" was indexed without position data; cannot run PhraseQuery (term=quick)]; }"));
SearchResponse searchResponse = client().prepareSearch().setQuery(matchQuery("field2", "quick brown").type(MatchQueryBuilder.Type.PHRASE).slop(0)).get();
assertHitCount(searchResponse, 1l);
try {
client().prepareSearch().setQuery(matchQuery("field1", "quick brown").type(MatchQueryBuilder.Type.PHRASE).slop(0)).get();
fail("SearchPhaseExecutionException should have been thrown");
} catch (SearchPhaseExecutionException e) {
assertTrue(e.getMessage().endsWith("IllegalStateException[field \"field1\" was indexed without position data; cannot run PhraseQuery (term=quick)]; }"));
}
wipeIndices("test");
} catch (MapperParsingException ex) {
assertThat(version.toString(), version.onOrAfter(Version.V_1_0_0), equalTo(true));
assertThat(ex.getCause().getMessage(), equalTo("'omit_term_freq_and_positions' is not supported anymore - use ['index_options' : 'DOCS_ONLY'] instead"));
}
version = randomVersion();
}
}