honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

uppercase first letter in property name breaks "_all" queries, closes #178.

This commit is contained in:
kimchy 2010-05-17 17:23:24 +03:00
parent d3881947ce
commit 23d2799d71
6 changed files with 49 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/XContentAllFieldMapper.java
View File

@ -20,7 +20,6 @@
package org.elasticsearch.index.mapper.xcontent;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.Fieldable;
import org.apache.lucene.index.Term;
@ -30,13 +29,12 @@ import org.elasticsearch.index.mapper.AllFieldMapper;
import org.elasticsearch.index.mapper.DocumentMapper;
import org.elasticsearch.index.mapper.MergeMappingException;
import org.elasticsearch.util.lucene.Lucene;
import org.elasticsearch.util.lucene.all.AllField;
import org.elasticsearch.util.lucene.all.AllTermQuery;
import org.elasticsearch.util.xcontent.builder.XContentBuilder;
import java.io.IOException;
import static org.elasticsearch.util.lucene.all.AllTokenFilter.*;
/**
* @author kimchy (shay.banon)
*/
@ -114,7 +112,7 @@ public class XContentAllFieldMapper extends XContentFieldMapper<Void> implements
return new AllTermQuery(new Term(names.indexName(), value));
}
@Override protected Field parseCreateField(ParseContext context) throws IOException {
@Override protected Fieldable parseCreateField(ParseContext context) throws IOException {
if (!enabled) {
return null;
}
@ -122,15 +120,7 @@ public class XContentAllFieldMapper extends XContentFieldMapper<Void> implements
context.allEntries().reset();
Analyzer analyzer = findAnalyzer(context.docMapper());
TokenStream tokenStream = allTokenStream(names.indexName(), context.allEntries(), analyzer);
if (stored()) {
// TODO when its possible to pass char[] to field, we can optimize
Field field = new Field(names.indexName(), context.allEntries().buildText(), store, index, termVector);
field.setTokenStream(tokenStream);
return field;
} else {
return new Field(names.indexName(), tokenStream, termVector);
}
return new AllField(names.indexName(), store, termVector, context.allEntries(), analyzer);
}
private Analyzer findAnalyzer(DocumentMapper docMapper) {

4
modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/XContentFieldMapper.java
View File

@ -267,7 +267,7 @@ public abstract class XContentFieldMapper<T> implements FieldMapper<T>, XContent
}
@Override public void parse(ParseContext context) throws IOException {
Field field = parseCreateField(context);
Fieldable field = parseCreateField(context);
if (field == null) {
return;
}
@ -279,7 +279,7 @@ public abstract class XContentFieldMapper<T> implements FieldMapper<T>, XContent
}
}
protected abstract Field parseCreateField(ParseContext context) throws IOException;
protected abstract Fieldable parseCreateField(ParseContext context) throws IOException;
@Override public void traverse(FieldMapperListener fieldMapperListener) {
fieldMapperListener.fieldMapper(this);

40
modules/elasticsearch/src/main/java/org/elasticsearch/util/lucene/all/AllAnalyzer.java → modules/elasticsearch/src/main/java/org/elasticsearch/util/lucene/all/AllField.java
View File

@ -21,39 +21,45 @@ package org.elasticsearch.util.lucene.all;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.document.Fieldable;
import org.apache.lucene.document.AbstractField;
import org.apache.lucene.document.Field;
import org.elasticsearch.ElasticSearchException;
import java.io.IOException;
import java.io.Reader;
/**
* An all analyzer.
*
* @author kimchy (shay.banon)
*/
public class AllAnalyzer extends Analyzer {
public class AllField extends AbstractField {
private final AllEntries allEntries;
private final Analyzer analyzer;
public AllAnalyzer(Analyzer analyzer) {
public AllField(String name, Field.Store store, Field.TermVector termVector, AllEntries allEntries, Analyzer analyzer) {
super(name, store, Field.Index.ANALYZED, termVector);
this.allEntries = allEntries;
this.analyzer = analyzer;
}
@Override public TokenStream tokenStream(String fieldName, Reader reader) {
AllEntries allEntries = (AllEntries) reader;
return new AllTokenFilter(analyzer.tokenStream(fieldName, reader), allEntries);
@Override public String stringValue() {
if (isStored()) {
return allEntries.buildText();
}
return null;
}
@Override public TokenStream reusableTokenStream(String fieldName, Reader reader) throws IOException {
AllEntries allEntries = (AllEntries) reader;
return new AllTokenFilter(analyzer.reusableTokenStream(fieldName, reader), allEntries);
@Override public Reader readerValue() {
return null;
}
@Override public int getPositionIncrementGap(String fieldName) {
return analyzer.getPositionIncrementGap(fieldName);
}
@Override public int getOffsetGap(Fieldable field) {
return analyzer.getOffsetGap(field);
@Override public TokenStream tokenStreamValue() {
try {
return AllTokenStream.allTokenStream(name, allEntries, analyzer);
} catch (IOException e) {
throw new ElasticSearchException("Failed to create token stream");
}
}
}

6
modules/elasticsearch/src/main/java/org/elasticsearch/util/lucene/all/AllTokenFilter.java → modules/elasticsearch/src/main/java/org/elasticsearch/util/lucene/all/AllTokenStream.java
View File

@ -32,17 +32,17 @@ import static org.apache.lucene.analysis.payloads.PayloadHelper.*;
/**
* @author kimchy (shay.banon)
*/
public class AllTokenFilter extends TokenFilter {
public class AllTokenStream extends TokenFilter {
public static TokenStream allTokenStream(String allFieldName, AllEntries allEntries, Analyzer analyzer) throws IOException {
return new AllTokenFilter(analyzer.reusableTokenStream(allFieldName, allEntries), allEntries);
return new AllTokenStream(analyzer.reusableTokenStream(allFieldName, allEntries), allEntries);
}
private final AllEntries allEntries;
private final PayloadAttribute payloadAttribute;
AllTokenFilter(TokenStream input, AllEntries allEntries) {
AllTokenStream(TokenStream input, AllEntries allEntries) {
super(input);
this.allEntries = allEntries;
payloadAttribute = addAttribute(PayloadAttribute.class);

20
modules/elasticsearch/src/test/java/org/elasticsearch/index/mapper/xcontent/all/SimpleAllMapperTests.java
View File

@ -20,13 +20,13 @@
package org.elasticsearch.index.mapper.xcontent.all;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.analysis.AnalysisService;
import org.elasticsearch.index.mapper.xcontent.XContentDocumentMapper;
import org.elasticsearch.index.mapper.xcontent.XContentDocumentMapperParser;
import org.elasticsearch.util.lucene.all.AllEntries;
import org.elasticsearch.util.lucene.all.AllTokenFilter;
import org.elasticsearch.util.lucene.all.AllField;
import org.elasticsearch.util.lucene.all.AllTokenStream;
import org.testng.annotations.Test;
import static org.elasticsearch.util.io.Streams.*;
@ -44,8 +44,8 @@ public class SimpleAllMapperTests {
XContentDocumentMapper docMapper = (XContentDocumentMapper) new XContentDocumentMapperParser(new AnalysisService(new Index("test"))).parse(mapping);
byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/xcontent/all/test1.json");
Document doc = docMapper.parse(json).doc();
Field field = doc.getField("_all");
AllEntries allEntries = ((AllTokenFilter) field.tokenStreamValue()).allEntries();
AllField field = (AllField) doc.getFieldable("_all");
AllEntries allEntries = ((AllTokenStream) field.tokenStreamValue()).allEntries();
assertThat(allEntries.fields().size(), equalTo(2));
assertThat(allEntries.fields().contains("name.last"), equalTo(true));
assertThat(allEntries.fields().contains("simple1"), equalTo(true));
@ -61,8 +61,8 @@ public class SimpleAllMapperTests {
byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/xcontent/all/test1.json");
Document doc = builtDocMapper.parse(json).doc();
Field field = doc.getField("_all");
AllEntries allEntries = ((AllTokenFilter) field.tokenStreamValue()).allEntries();
AllField field = (AllField) doc.getFieldable("_all");
AllEntries allEntries = ((AllTokenStream) field.tokenStreamValue()).allEntries();
assertThat(allEntries.fields().size(), equalTo(2));
assertThat(allEntries.fields().contains("name.last"), equalTo(true));
assertThat(allEntries.fields().contains("simple1"), equalTo(true));
@ -73,8 +73,8 @@ public class SimpleAllMapperTests {
XContentDocumentMapper docMapper = (XContentDocumentMapper) new XContentDocumentMapperParser(new AnalysisService(new Index("test"))).parse(mapping);
byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/xcontent/all/test1.json");
Document doc = docMapper.parse(json).doc();
Field field = doc.getField("_all");
AllEntries allEntries = ((AllTokenFilter) field.tokenStreamValue()).allEntries();
AllField field = (AllField) doc.getFieldable("_all");
AllEntries allEntries = ((AllTokenStream) field.tokenStreamValue()).allEntries();
assertThat(allEntries.fields().size(), equalTo(2));
assertThat(allEntries.fields().contains("name.last"), equalTo(true));
assertThat(allEntries.fields().contains("simple1"), equalTo(true));
@ -93,8 +93,8 @@ public class SimpleAllMapperTests {
byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/xcontent/all/test1.json");
Document doc = builtDocMapper.parse(json).doc();
Field field = doc.getField("_all");
AllEntries allEntries = ((AllTokenFilter) field.tokenStreamValue()).allEntries();
AllField field = (AllField) doc.getFieldable("_all");
AllEntries allEntries = ((AllTokenStream) field.tokenStreamValue()).allEntries();
assertThat(allEntries.fields().size(), equalTo(2));
assertThat(allEntries.fields().contains("name.last"), equalTo(true));
assertThat(allEntries.fields().contains("simple1"), equalTo(true));

16
modules/elasticsearch/src/test/java/org/elasticsearch/util/lucene/all/SimpleAllTests.java
View File

@ -50,7 +50,7 @@ public class SimpleAllTests {
allEntries.addText("field1", "something", 1.0f);
allEntries.addText("field2", "else", 1.0f);
allEntries.reset();
doc.add(new Field("_all", AllTokenFilter.allTokenStream("_all", allEntries, Lucene.STANDARD_ANALYZER)));
doc.add(new Field("_all", AllTokenStream.allTokenStream("_all", allEntries, Lucene.STANDARD_ANALYZER)));
indexWriter.addDocument(doc);
@ -60,7 +60,7 @@ public class SimpleAllTests {
allEntries.addText("field1", "else", 1.0f);
allEntries.addText("field2", "something", 1.0f);
allEntries.reset();
doc.add(new Field("_all", AllTokenFilter.allTokenStream("_all", allEntries, Lucene.STANDARD_ANALYZER)));
doc.add(new Field("_all", AllTokenStream.allTokenStream("_all", allEntries, Lucene.STANDARD_ANALYZER)));
indexWriter.addDocument(doc);
@ -92,7 +92,7 @@ public class SimpleAllTests {
allEntries.addText("field1", "something", 1.0f);
allEntries.addText("field2", "else", 1.0f);
allEntries.reset();
doc.add(new Field("_all", AllTokenFilter.allTokenStream("_all", allEntries, Lucene.STANDARD_ANALYZER)));
doc.add(new Field("_all", AllTokenStream.allTokenStream("_all", allEntries, Lucene.STANDARD_ANALYZER)));
indexWriter.addDocument(doc);
@ -102,7 +102,7 @@ public class SimpleAllTests {
allEntries.addText("field1", "else", 2.0f);
allEntries.addText("field2", "something", 1.0f);
allEntries.reset();
doc.add(new Field("_all", AllTokenFilter.allTokenStream("_all", allEntries, Lucene.STANDARD_ANALYZER)));
doc.add(new Field("_all", AllTokenStream.allTokenStream("_all", allEntries, Lucene.STANDARD_ANALYZER)));
indexWriter.addDocument(doc);
@ -135,7 +135,7 @@ public class SimpleAllTests {
allEntries.addText("field1", "something moo", 1.0f);
allEntries.addText("field2", "else koo", 1.0f);
allEntries.reset();
doc.add(new Field("_all", AllTokenFilter.allTokenStream("_all", allEntries, Lucene.STANDARD_ANALYZER)));
doc.add(new Field("_all", AllTokenStream.allTokenStream("_all", allEntries, Lucene.STANDARD_ANALYZER)));
indexWriter.addDocument(doc);
@ -145,7 +145,7 @@ public class SimpleAllTests {
allEntries.addText("field1", "else koo", 1.0f);
allEntries.addText("field2", "something moo", 1.0f);
allEntries.reset();
doc.add(new Field("_all", AllTokenFilter.allTokenStream("_all", allEntries, Lucene.STANDARD_ANALYZER)));
doc.add(new Field("_all", AllTokenStream.allTokenStream("_all", allEntries, Lucene.STANDARD_ANALYZER)));
indexWriter.addDocument(doc);
@ -187,7 +187,7 @@ public class SimpleAllTests {
allEntries.addText("field1", "something moo", 1.0f);
allEntries.addText("field2", "else koo", 1.0f);
allEntries.reset();
doc.add(new Field("_all", AllTokenFilter.allTokenStream("_all", allEntries, Lucene.STANDARD_ANALYZER)));
doc.add(new Field("_all", AllTokenStream.allTokenStream("_all", allEntries, Lucene.STANDARD_ANALYZER)));
indexWriter.addDocument(doc);
@ -197,7 +197,7 @@ public class SimpleAllTests {
allEntries.addText("field1", "else koo", 2.0f);
allEntries.addText("field2", "something moo", 1.0f);
allEntries.reset();
doc.add(new Field("_all", AllTokenFilter.allTokenStream("_all", allEntries, Lucene.STANDARD_ANALYZER)));
doc.add(new Field("_all", AllTokenStream.allTokenStream("_all", allEntries, Lucene.STANDARD_ANALYZER)));
indexWriter.addDocument(doc);