honeymoose/OpenSearch
1
0
Fork 0
mirror of https://github.com/honeymoose/OpenSearch.git synced 2025-02-09 22:45:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

uppercase first letter in property name breaks "_all" queries, closes #178.

Browse Source
This commit is contained in:
kimchy 2010-05-17 17:23:24 +03:00
parent d3881947ce
commit 23d2799d71
6 changed files with 49 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/XContentAllFieldMapper.java
View File

@ -20,7 +20,6 @@
package org.elasticsearch.index.mapper.xcontent; package org.elasticsearch.index.mapper.xcontent;
import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.document.Field; import org.apache.lucene.document.Field;
import org.apache.lucene.document.Fieldable; import org.apache.lucene.document.Fieldable;
import org.apache.lucene.index.Term; import org.apache.lucene.index.Term;
@ -30,13 +29,12 @@ import org.elasticsearch.index.mapper.AllFieldMapper;
import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.DocumentMapper;
import org.elasticsearch.index.mapper.MergeMappingException; import org.elasticsearch.index.mapper.MergeMappingException;
import org.elasticsearch.util.lucene.Lucene; import org.elasticsearch.util.lucene.Lucene;
import org.elasticsearch.util.lucene.all.AllField;
import org.elasticsearch.util.lucene.all.AllTermQuery; import org.elasticsearch.util.lucene.all.AllTermQuery;
import org.elasticsearch.util.xcontent.builder.XContentBuilder; import org.elasticsearch.util.xcontent.builder.XContentBuilder;
import java.io.IOException; import java.io.IOException;
import static org.elasticsearch.util.lucene.all.AllTokenFilter.*;
/** /**
* @author kimchy (shay.banon) * @author kimchy (shay.banon)
*/ */
@ -114,7 +112,7 @@ public class XContentAllFieldMapper extends XContentFieldMapper<Void> implements
return new AllTermQuery(new Term(names.indexName(), value)); return new AllTermQuery(new Term(names.indexName(), value));
} }
@Override protected Field parseCreateField(ParseContext context) throws IOException { @Override protected Fieldable parseCreateField(ParseContext context) throws IOException {
if (!enabled) { if (!enabled) {
return null; return null;
} }
@ -122,15 +120,7 @@ public class XContentAllFieldMapper extends XContentFieldMapper<Void> implements
context.allEntries().reset(); context.allEntries().reset();
Analyzer analyzer = findAnalyzer(context.docMapper()); Analyzer analyzer = findAnalyzer(context.docMapper());
TokenStream tokenStream = allTokenStream(names.indexName(), context.allEntries(), analyzer); return new AllField(names.indexName(), store, termVector, context.allEntries(), analyzer);
if (stored()) {
// TODO when its possible to pass char[] to field, we can optimize
Field field = new Field(names.indexName(), context.allEntries().buildText(), store, index, termVector);
field.setTokenStream(tokenStream);
return field;
} else {
return new Field(names.indexName(), tokenStream, termVector);
}
} }
private Analyzer findAnalyzer(DocumentMapper docMapper) { private Analyzer findAnalyzer(DocumentMapper docMapper) {

4
modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/XContentFieldMapper.java
View File

@ -267,7 +267,7 @@ public abstract class XContentFieldMapper<T> implements FieldMapper<T>, XContent
} }
@Override public void parse(ParseContext context) throws IOException { @Override public void parse(ParseContext context) throws IOException {
Field field = parseCreateField(context); Fieldable field = parseCreateField(context);
if (field == null) { if (field == null) {
return; return;
} }
@ -279,7 +279,7 @@ public abstract class XContentFieldMapper<T> implements FieldMapper<T>, XContent
} }
} }
protected abstract Field parseCreateField(ParseContext context) throws IOException; protected abstract Fieldable parseCreateField(ParseContext context) throws IOException;
@Override public void traverse(FieldMapperListener fieldMapperListener) { @Override public void traverse(FieldMapperListener fieldMapperListener) {
fieldMapperListener.fieldMapper(this); fieldMapperListener.fieldMapper(this);

40
modules/elasticsearch/src/main/java/org/elasticsearch/util/lucene/all/AllAnalyzer.java → modules/elasticsearch/src/main/java/org/elasticsearch/util/lucene/all/AllField.java
View File

@ -21,39 +21,45 @@ package org.elasticsearch.util.lucene.all;
import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.document.Fieldable; import org.apache.lucene.document.AbstractField;
import org.apache.lucene.document.Field;
import org.elasticsearch.ElasticSearchException;
import java.io.IOException; import java.io.IOException;
import java.io.Reader; import java.io.Reader;
/** /**
* An all analyzer.
*
* @author kimchy (shay.banon) * @author kimchy (shay.banon)
*/ */
public class AllAnalyzer extends Analyzer { public class AllField extends AbstractField {
private final AllEntries allEntries;
private final Analyzer analyzer; private final Analyzer analyzer;
public AllAnalyzer(Analyzer analyzer) { public AllField(String name, Field.Store store, Field.TermVector termVector, AllEntries allEntries, Analyzer analyzer) {
super(name, store, Field.Index.ANALYZED, termVector);
this.allEntries = allEntries;
this.analyzer = analyzer; this.analyzer = analyzer;
} }
@Override public TokenStream tokenStream(String fieldName, Reader reader) { @Override public String stringValue() {
AllEntries allEntries = (AllEntries) reader; if (isStored()) {
return new AllTokenFilter(analyzer.tokenStream(fieldName, reader), allEntries); return allEntries.buildText();
}
return null;
} }
@Override public TokenStream reusableTokenStream(String fieldName, Reader reader) throws IOException { @Override public Reader readerValue() {
AllEntries allEntries = (AllEntries) reader; return null;
return new AllTokenFilter(analyzer.reusableTokenStream(fieldName, reader), allEntries);
} }
@Override public int getPositionIncrementGap(String fieldName) { @Override public TokenStream tokenStreamValue() {
return analyzer.getPositionIncrementGap(fieldName); try {
} return AllTokenStream.allTokenStream(name, allEntries, analyzer);
} catch (IOException e) {
@Override public int getOffsetGap(Fieldable field) { throw new ElasticSearchException("Failed to create token stream");
return analyzer.getOffsetGap(field); }
} }
} }

6
modules/elasticsearch/src/main/java/org/elasticsearch/util/lucene/all/AllTokenFilter.java → modules/elasticsearch/src/main/java/org/elasticsearch/util/lucene/all/AllTokenStream.java
View File

@ -32,17 +32,17 @@ import static org.apache.lucene.analysis.payloads.PayloadHelper.*;
/** /**
* @author kimchy (shay.banon) * @author kimchy (shay.banon)
*/ */
public class AllTokenFilter extends TokenFilter { public class AllTokenStream extends TokenFilter {
public static TokenStream allTokenStream(String allFieldName, AllEntries allEntries, Analyzer analyzer) throws IOException { public static TokenStream allTokenStream(String allFieldName, AllEntries allEntries, Analyzer analyzer) throws IOException {
return new AllTokenFilter(analyzer.reusableTokenStream(allFieldName, allEntries), allEntries); return new AllTokenStream(analyzer.reusableTokenStream(allFieldName, allEntries), allEntries);
} }
private final AllEntries allEntries; private final AllEntries allEntries;
private final PayloadAttribute payloadAttribute; private final PayloadAttribute payloadAttribute;
AllTokenFilter(TokenStream input, AllEntries allEntries) { AllTokenStream(TokenStream input, AllEntries allEntries) {
super(input); super(input);
this.allEntries = allEntries; this.allEntries = allEntries;
payloadAttribute = addAttribute(PayloadAttribute.class); payloadAttribute = addAttribute(PayloadAttribute.class);

20
modules/elasticsearch/src/test/java/org/elasticsearch/index/mapper/xcontent/all/SimpleAllMapperTests.java
View File

@ -20,13 +20,13 @@
package org.elasticsearch.index.mapper.xcontent.all; package org.elasticsearch.index.mapper.xcontent.all;
import org.apache.lucene.document.Document; import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.elasticsearch.index.Index; import org.elasticsearch.index.Index;
import org.elasticsearch.index.analysis.AnalysisService; import org.elasticsearch.index.analysis.AnalysisService;
import org.elasticsearch.index.mapper.xcontent.XContentDocumentMapper; import org.elasticsearch.index.mapper.xcontent.XContentDocumentMapper;
import org.elasticsearch.index.mapper.xcontent.XContentDocumentMapperParser; import org.elasticsearch.index.mapper.xcontent.XContentDocumentMapperParser;
import org.elasticsearch.util.lucene.all.AllEntries; import org.elasticsearch.util.lucene.all.AllEntries;
import org.elasticsearch.util.lucene.all.AllTokenFilter; import org.elasticsearch.util.lucene.all.AllField;
import org.elasticsearch.util.lucene.all.AllTokenStream;
import org.testng.annotations.Test; import org.testng.annotations.Test;
import static org.elasticsearch.util.io.Streams.*; import static org.elasticsearch.util.io.Streams.*;
@ -44,8 +44,8 @@ public class SimpleAllMapperTests {
XContentDocumentMapper docMapper = (XContentDocumentMapper) new XContentDocumentMapperParser(new AnalysisService(new Index("test"))).parse(mapping); XContentDocumentMapper docMapper = (XContentDocumentMapper) new XContentDocumentMapperParser(new AnalysisService(new Index("test"))).parse(mapping);
byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/xcontent/all/test1.json"); byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/xcontent/all/test1.json");
Document doc = docMapper.parse(json).doc(); Document doc = docMapper.parse(json).doc();
Field field = doc.getField("_all"); AllField field = (AllField) doc.getFieldable("_all");
AllEntries allEntries = ((AllTokenFilter) field.tokenStreamValue()).allEntries(); AllEntries allEntries = ((AllTokenStream) field.tokenStreamValue()).allEntries();
assertThat(allEntries.fields().size(), equalTo(2)); assertThat(allEntries.fields().size(), equalTo(2));
assertThat(allEntries.fields().contains("name.last"), equalTo(true)); assertThat(allEntries.fields().contains("name.last"), equalTo(true));
assertThat(allEntries.fields().contains("simple1"), equalTo(true)); assertThat(allEntries.fields().contains("simple1"), equalTo(true));
@ -61,8 +61,8 @@ public class SimpleAllMapperTests {
byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/xcontent/all/test1.json"); byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/xcontent/all/test1.json");
Document doc = builtDocMapper.parse(json).doc(); Document doc = builtDocMapper.parse(json).doc();
Field field = doc.getField("_all"); AllField field = (AllField) doc.getFieldable("_all");
AllEntries allEntries = ((AllTokenFilter) field.tokenStreamValue()).allEntries(); AllEntries allEntries = ((AllTokenStream) field.tokenStreamValue()).allEntries();
assertThat(allEntries.fields().size(), equalTo(2)); assertThat(allEntries.fields().size(), equalTo(2));
assertThat(allEntries.fields().contains("name.last"), equalTo(true)); assertThat(allEntries.fields().contains("name.last"), equalTo(true));
assertThat(allEntries.fields().contains("simple1"), equalTo(true)); assertThat(allEntries.fields().contains("simple1"), equalTo(true));
@ -73,8 +73,8 @@ public class SimpleAllMapperTests {
XContentDocumentMapper docMapper = (XContentDocumentMapper) new XContentDocumentMapperParser(new AnalysisService(new Index("test"))).parse(mapping); XContentDocumentMapper docMapper = (XContentDocumentMapper) new XContentDocumentMapperParser(new AnalysisService(new Index("test"))).parse(mapping);
byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/xcontent/all/test1.json"); byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/xcontent/all/test1.json");
Document doc = docMapper.parse(json).doc(); Document doc = docMapper.parse(json).doc();
Field field = doc.getField("_all"); AllField field = (AllField) doc.getFieldable("_all");
AllEntries allEntries = ((AllTokenFilter) field.tokenStreamValue()).allEntries(); AllEntries allEntries = ((AllTokenStream) field.tokenStreamValue()).allEntries();
assertThat(allEntries.fields().size(), equalTo(2)); assertThat(allEntries.fields().size(), equalTo(2));
assertThat(allEntries.fields().contains("name.last"), equalTo(true)); assertThat(allEntries.fields().contains("name.last"), equalTo(true));
assertThat(allEntries.fields().contains("simple1"), equalTo(true)); assertThat(allEntries.fields().contains("simple1"), equalTo(true));
@ -93,8 +93,8 @@ public class SimpleAllMapperTests {
byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/xcontent/all/test1.json"); byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/xcontent/all/test1.json");
Document doc = builtDocMapper.parse(json).doc(); Document doc = builtDocMapper.parse(json).doc();
Field field = doc.getField("_all"); AllField field = (AllField) doc.getFieldable("_all");
AllEntries allEntries = ((AllTokenFilter) field.tokenStreamValue()).allEntries(); AllEntries allEntries = ((AllTokenStream) field.tokenStreamValue()).allEntries();
assertThat(allEntries.fields().size(), equalTo(2)); assertThat(allEntries.fields().size(), equalTo(2));
assertThat(allEntries.fields().contains("name.last"), equalTo(true)); assertThat(allEntries.fields().contains("name.last"), equalTo(true));
assertThat(allEntries.fields().contains("simple1"), equalTo(true)); assertThat(allEntries.fields().contains("simple1"), equalTo(true));

16
modules/elasticsearch/src/test/java/org/elasticsearch/util/lucene/all/SimpleAllTests.java
View File

@ -50,7 +50,7 @@ public class SimpleAllTests {
allEntries.addText("field1", "something", 1.0f); allEntries.addText("field1", "something", 1.0f);
allEntries.addText("field2", "else", 1.0f); allEntries.addText("field2", "else", 1.0f);
allEntries.reset(); allEntries.reset();
doc.add(new Field("_all", AllTokenFilter.allTokenStream("_all", allEntries, Lucene.STANDARD_ANALYZER))); doc.add(new Field("_all", AllTokenStream.allTokenStream("_all", allEntries, Lucene.STANDARD_ANALYZER)));
indexWriter.addDocument(doc); indexWriter.addDocument(doc);
@ -60,7 +60,7 @@ public class SimpleAllTests {
allEntries.addText("field1", "else", 1.0f); allEntries.addText("field1", "else", 1.0f);
allEntries.addText("field2", "something", 1.0f); allEntries.addText("field2", "something", 1.0f);
allEntries.reset(); allEntries.reset();
doc.add(new Field("_all", AllTokenFilter.allTokenStream("_all", allEntries, Lucene.STANDARD_ANALYZER))); doc.add(new Field("_all", AllTokenStream.allTokenStream("_all", allEntries, Lucene.STANDARD_ANALYZER)));
indexWriter.addDocument(doc); indexWriter.addDocument(doc);
@ -92,7 +92,7 @@ public class SimpleAllTests {
allEntries.addText("field1", "something", 1.0f); allEntries.addText("field1", "something", 1.0f);
allEntries.addText("field2", "else", 1.0f); allEntries.addText("field2", "else", 1.0f);
allEntries.reset(); allEntries.reset();
doc.add(new Field("_all", AllTokenFilter.allTokenStream("_all", allEntries, Lucene.STANDARD_ANALYZER))); doc.add(new Field("_all", AllTokenStream.allTokenStream("_all", allEntries, Lucene.STANDARD_ANALYZER)));
indexWriter.addDocument(doc); indexWriter.addDocument(doc);
@ -102,7 +102,7 @@ public class SimpleAllTests {
allEntries.addText("field1", "else", 2.0f); allEntries.addText("field1", "else", 2.0f);
allEntries.addText("field2", "something", 1.0f); allEntries.addText("field2", "something", 1.0f);
allEntries.reset(); allEntries.reset();
doc.add(new Field("_all", AllTokenFilter.allTokenStream("_all", allEntries, Lucene.STANDARD_ANALYZER))); doc.add(new Field("_all", AllTokenStream.allTokenStream("_all", allEntries, Lucene.STANDARD_ANALYZER)));
indexWriter.addDocument(doc); indexWriter.addDocument(doc);
@ -135,7 +135,7 @@ public class SimpleAllTests {
allEntries.addText("field1", "something moo", 1.0f); allEntries.addText("field1", "something moo", 1.0f);
allEntries.addText("field2", "else koo", 1.0f); allEntries.addText("field2", "else koo", 1.0f);
allEntries.reset(); allEntries.reset();
doc.add(new Field("_all", AllTokenFilter.allTokenStream("_all", allEntries, Lucene.STANDARD_ANALYZER))); doc.add(new Field("_all", AllTokenStream.allTokenStream("_all", allEntries, Lucene.STANDARD_ANALYZER)));
indexWriter.addDocument(doc); indexWriter.addDocument(doc);
@ -145,7 +145,7 @@ public class SimpleAllTests {
allEntries.addText("field1", "else koo", 1.0f); allEntries.addText("field1", "else koo", 1.0f);
allEntries.addText("field2", "something moo", 1.0f); allEntries.addText("field2", "something moo", 1.0f);
allEntries.reset(); allEntries.reset();
doc.add(new Field("_all", AllTokenFilter.allTokenStream("_all", allEntries, Lucene.STANDARD_ANALYZER))); doc.add(new Field("_all", AllTokenStream.allTokenStream("_all", allEntries, Lucene.STANDARD_ANALYZER)));
indexWriter.addDocument(doc); indexWriter.addDocument(doc);
@ -187,7 +187,7 @@ public class SimpleAllTests {
allEntries.addText("field1", "something moo", 1.0f); allEntries.addText("field1", "something moo", 1.0f);
allEntries.addText("field2", "else koo", 1.0f); allEntries.addText("field2", "else koo", 1.0f);
allEntries.reset(); allEntries.reset();
doc.add(new Field("_all", AllTokenFilter.allTokenStream("_all", allEntries, Lucene.STANDARD_ANALYZER))); doc.add(new Field("_all", AllTokenStream.allTokenStream("_all", allEntries, Lucene.STANDARD_ANALYZER)));
indexWriter.addDocument(doc); indexWriter.addDocument(doc);
@ -197,7 +197,7 @@ public class SimpleAllTests {
allEntries.addText("field1", "else koo", 2.0f); allEntries.addText("field1", "else koo", 2.0f);
allEntries.addText("field2", "something moo", 1.0f); allEntries.addText("field2", "something moo", 1.0f);
allEntries.reset(); allEntries.reset();
doc.add(new Field("_all", AllTokenFilter.allTokenStream("_all", allEntries, Lucene.STANDARD_ANALYZER))); doc.add(new Field("_all", AllTokenStream.allTokenStream("_all", allEntries, Lucene.STANDARD_ANALYZER)));
indexWriter.addDocument(doc); indexWriter.addDocument(doc);