diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/XContentAllFieldMapper.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/XContentAllFieldMapper.java index 2bf874b0d56..ffa20a0c5bd 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/XContentAllFieldMapper.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/XContentAllFieldMapper.java @@ -20,7 +20,6 @@ package org.elasticsearch.index.mapper.xcontent; import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.document.Field; import org.apache.lucene.document.Fieldable; import org.apache.lucene.index.Term; @@ -30,13 +29,12 @@ import org.elasticsearch.index.mapper.AllFieldMapper; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.MergeMappingException; import org.elasticsearch.util.lucene.Lucene; +import org.elasticsearch.util.lucene.all.AllField; import org.elasticsearch.util.lucene.all.AllTermQuery; import org.elasticsearch.util.xcontent.builder.XContentBuilder; import java.io.IOException; -import static org.elasticsearch.util.lucene.all.AllTokenFilter.*; - /** * @author kimchy (shay.banon) */ @@ -114,7 +112,7 @@ public class XContentAllFieldMapper extends XContentFieldMapper implements return new AllTermQuery(new Term(names.indexName(), value)); } - @Override protected Field parseCreateField(ParseContext context) throws IOException { + @Override protected Fieldable parseCreateField(ParseContext context) throws IOException { if (!enabled) { return null; } @@ -122,15 +120,7 @@ public class XContentAllFieldMapper extends XContentFieldMapper implements context.allEntries().reset(); Analyzer analyzer = findAnalyzer(context.docMapper()); - TokenStream tokenStream = allTokenStream(names.indexName(), context.allEntries(), analyzer); - if (stored()) { - // TODO when its possible to pass char[] to field, we can optimize - Field field = new Field(names.indexName(), context.allEntries().buildText(), store, index, termVector); - field.setTokenStream(tokenStream); - return field; - } else { - return new Field(names.indexName(), tokenStream, termVector); - } + return new AllField(names.indexName(), store, termVector, context.allEntries(), analyzer); } private Analyzer findAnalyzer(DocumentMapper docMapper) { diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/XContentFieldMapper.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/XContentFieldMapper.java index 1c243bf2b8c..47471eca52b 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/XContentFieldMapper.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/XContentFieldMapper.java @@ -267,7 +267,7 @@ public abstract class XContentFieldMapper implements FieldMapper, XContent } @Override public void parse(ParseContext context) throws IOException { - Field field = parseCreateField(context); + Fieldable field = parseCreateField(context); if (field == null) { return; } @@ -279,7 +279,7 @@ public abstract class XContentFieldMapper implements FieldMapper, XContent } } - protected abstract Field parseCreateField(ParseContext context) throws IOException; + protected abstract Fieldable parseCreateField(ParseContext context) throws IOException; @Override public void traverse(FieldMapperListener fieldMapperListener) { fieldMapperListener.fieldMapper(this); diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/util/lucene/all/AllAnalyzer.java b/modules/elasticsearch/src/main/java/org/elasticsearch/util/lucene/all/AllField.java similarity index 55% rename from modules/elasticsearch/src/main/java/org/elasticsearch/util/lucene/all/AllAnalyzer.java rename to modules/elasticsearch/src/main/java/org/elasticsearch/util/lucene/all/AllField.java index af4184bcfa8..18481ad8004 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/util/lucene/all/AllAnalyzer.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/util/lucene/all/AllField.java @@ -21,39 +21,45 @@ package org.elasticsearch.util.lucene.all; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.TokenStream; -import org.apache.lucene.document.Fieldable; +import org.apache.lucene.document.AbstractField; +import org.apache.lucene.document.Field; +import org.elasticsearch.ElasticSearchException; import java.io.IOException; import java.io.Reader; /** - * An all analyzer. - * * @author kimchy (shay.banon) */ -public class AllAnalyzer extends Analyzer { +public class AllField extends AbstractField { + + private final AllEntries allEntries; private final Analyzer analyzer; - public AllAnalyzer(Analyzer analyzer) { + public AllField(String name, Field.Store store, Field.TermVector termVector, AllEntries allEntries, Analyzer analyzer) { + super(name, store, Field.Index.ANALYZED, termVector); + + this.allEntries = allEntries; this.analyzer = analyzer; } - @Override public TokenStream tokenStream(String fieldName, Reader reader) { - AllEntries allEntries = (AllEntries) reader; - return new AllTokenFilter(analyzer.tokenStream(fieldName, reader), allEntries); + @Override public String stringValue() { + if (isStored()) { + return allEntries.buildText(); + } + return null; } - @Override public TokenStream reusableTokenStream(String fieldName, Reader reader) throws IOException { - AllEntries allEntries = (AllEntries) reader; - return new AllTokenFilter(analyzer.reusableTokenStream(fieldName, reader), allEntries); + @Override public Reader readerValue() { + return null; } - @Override public int getPositionIncrementGap(String fieldName) { - return analyzer.getPositionIncrementGap(fieldName); - } - - @Override public int getOffsetGap(Fieldable field) { - return analyzer.getOffsetGap(field); + @Override public TokenStream tokenStreamValue() { + try { + return AllTokenStream.allTokenStream(name, allEntries, analyzer); + } catch (IOException e) { + throw new ElasticSearchException("Failed to create token stream"); + } } } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/util/lucene/all/AllTokenFilter.java b/modules/elasticsearch/src/main/java/org/elasticsearch/util/lucene/all/AllTokenStream.java similarity index 92% rename from modules/elasticsearch/src/main/java/org/elasticsearch/util/lucene/all/AllTokenFilter.java rename to modules/elasticsearch/src/main/java/org/elasticsearch/util/lucene/all/AllTokenStream.java index 5add3b8080d..e66d4ba2eb5 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/util/lucene/all/AllTokenFilter.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/util/lucene/all/AllTokenStream.java @@ -32,17 +32,17 @@ import static org.apache.lucene.analysis.payloads.PayloadHelper.*; /** * @author kimchy (shay.banon) */ -public class AllTokenFilter extends TokenFilter { +public class AllTokenStream extends TokenFilter { public static TokenStream allTokenStream(String allFieldName, AllEntries allEntries, Analyzer analyzer) throws IOException { - return new AllTokenFilter(analyzer.reusableTokenStream(allFieldName, allEntries), allEntries); + return new AllTokenStream(analyzer.reusableTokenStream(allFieldName, allEntries), allEntries); } private final AllEntries allEntries; private final PayloadAttribute payloadAttribute; - AllTokenFilter(TokenStream input, AllEntries allEntries) { + AllTokenStream(TokenStream input, AllEntries allEntries) { super(input); this.allEntries = allEntries; payloadAttribute = addAttribute(PayloadAttribute.class); diff --git a/modules/elasticsearch/src/test/java/org/elasticsearch/index/mapper/xcontent/all/SimpleAllMapperTests.java b/modules/elasticsearch/src/test/java/org/elasticsearch/index/mapper/xcontent/all/SimpleAllMapperTests.java index b8b38b30cdc..10479f52771 100644 --- a/modules/elasticsearch/src/test/java/org/elasticsearch/index/mapper/xcontent/all/SimpleAllMapperTests.java +++ b/modules/elasticsearch/src/test/java/org/elasticsearch/index/mapper/xcontent/all/SimpleAllMapperTests.java @@ -20,13 +20,13 @@ package org.elasticsearch.index.mapper.xcontent.all; import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field; import org.elasticsearch.index.Index; import org.elasticsearch.index.analysis.AnalysisService; import org.elasticsearch.index.mapper.xcontent.XContentDocumentMapper; import org.elasticsearch.index.mapper.xcontent.XContentDocumentMapperParser; import org.elasticsearch.util.lucene.all.AllEntries; -import org.elasticsearch.util.lucene.all.AllTokenFilter; +import org.elasticsearch.util.lucene.all.AllField; +import org.elasticsearch.util.lucene.all.AllTokenStream; import org.testng.annotations.Test; import static org.elasticsearch.util.io.Streams.*; @@ -44,8 +44,8 @@ public class SimpleAllMapperTests { XContentDocumentMapper docMapper = (XContentDocumentMapper) new XContentDocumentMapperParser(new AnalysisService(new Index("test"))).parse(mapping); byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/xcontent/all/test1.json"); Document doc = docMapper.parse(json).doc(); - Field field = doc.getField("_all"); - AllEntries allEntries = ((AllTokenFilter) field.tokenStreamValue()).allEntries(); + AllField field = (AllField) doc.getFieldable("_all"); + AllEntries allEntries = ((AllTokenStream) field.tokenStreamValue()).allEntries(); assertThat(allEntries.fields().size(), equalTo(2)); assertThat(allEntries.fields().contains("name.last"), equalTo(true)); assertThat(allEntries.fields().contains("simple1"), equalTo(true)); @@ -61,8 +61,8 @@ public class SimpleAllMapperTests { byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/xcontent/all/test1.json"); Document doc = builtDocMapper.parse(json).doc(); - Field field = doc.getField("_all"); - AllEntries allEntries = ((AllTokenFilter) field.tokenStreamValue()).allEntries(); + AllField field = (AllField) doc.getFieldable("_all"); + AllEntries allEntries = ((AllTokenStream) field.tokenStreamValue()).allEntries(); assertThat(allEntries.fields().size(), equalTo(2)); assertThat(allEntries.fields().contains("name.last"), equalTo(true)); assertThat(allEntries.fields().contains("simple1"), equalTo(true)); @@ -73,8 +73,8 @@ public class SimpleAllMapperTests { XContentDocumentMapper docMapper = (XContentDocumentMapper) new XContentDocumentMapperParser(new AnalysisService(new Index("test"))).parse(mapping); byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/xcontent/all/test1.json"); Document doc = docMapper.parse(json).doc(); - Field field = doc.getField("_all"); - AllEntries allEntries = ((AllTokenFilter) field.tokenStreamValue()).allEntries(); + AllField field = (AllField) doc.getFieldable("_all"); + AllEntries allEntries = ((AllTokenStream) field.tokenStreamValue()).allEntries(); assertThat(allEntries.fields().size(), equalTo(2)); assertThat(allEntries.fields().contains("name.last"), equalTo(true)); assertThat(allEntries.fields().contains("simple1"), equalTo(true)); @@ -93,8 +93,8 @@ public class SimpleAllMapperTests { byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/xcontent/all/test1.json"); Document doc = builtDocMapper.parse(json).doc(); - Field field = doc.getField("_all"); - AllEntries allEntries = ((AllTokenFilter) field.tokenStreamValue()).allEntries(); + AllField field = (AllField) doc.getFieldable("_all"); + AllEntries allEntries = ((AllTokenStream) field.tokenStreamValue()).allEntries(); assertThat(allEntries.fields().size(), equalTo(2)); assertThat(allEntries.fields().contains("name.last"), equalTo(true)); assertThat(allEntries.fields().contains("simple1"), equalTo(true)); diff --git a/modules/elasticsearch/src/test/java/org/elasticsearch/util/lucene/all/SimpleAllTests.java b/modules/elasticsearch/src/test/java/org/elasticsearch/util/lucene/all/SimpleAllTests.java index 954c08a1c34..e54bff3a56c 100644 --- a/modules/elasticsearch/src/test/java/org/elasticsearch/util/lucene/all/SimpleAllTests.java +++ b/modules/elasticsearch/src/test/java/org/elasticsearch/util/lucene/all/SimpleAllTests.java @@ -50,7 +50,7 @@ public class SimpleAllTests { allEntries.addText("field1", "something", 1.0f); allEntries.addText("field2", "else", 1.0f); allEntries.reset(); - doc.add(new Field("_all", AllTokenFilter.allTokenStream("_all", allEntries, Lucene.STANDARD_ANALYZER))); + doc.add(new Field("_all", AllTokenStream.allTokenStream("_all", allEntries, Lucene.STANDARD_ANALYZER))); indexWriter.addDocument(doc); @@ -60,7 +60,7 @@ public class SimpleAllTests { allEntries.addText("field1", "else", 1.0f); allEntries.addText("field2", "something", 1.0f); allEntries.reset(); - doc.add(new Field("_all", AllTokenFilter.allTokenStream("_all", allEntries, Lucene.STANDARD_ANALYZER))); + doc.add(new Field("_all", AllTokenStream.allTokenStream("_all", allEntries, Lucene.STANDARD_ANALYZER))); indexWriter.addDocument(doc); @@ -92,7 +92,7 @@ public class SimpleAllTests { allEntries.addText("field1", "something", 1.0f); allEntries.addText("field2", "else", 1.0f); allEntries.reset(); - doc.add(new Field("_all", AllTokenFilter.allTokenStream("_all", allEntries, Lucene.STANDARD_ANALYZER))); + doc.add(new Field("_all", AllTokenStream.allTokenStream("_all", allEntries, Lucene.STANDARD_ANALYZER))); indexWriter.addDocument(doc); @@ -102,7 +102,7 @@ public class SimpleAllTests { allEntries.addText("field1", "else", 2.0f); allEntries.addText("field2", "something", 1.0f); allEntries.reset(); - doc.add(new Field("_all", AllTokenFilter.allTokenStream("_all", allEntries, Lucene.STANDARD_ANALYZER))); + doc.add(new Field("_all", AllTokenStream.allTokenStream("_all", allEntries, Lucene.STANDARD_ANALYZER))); indexWriter.addDocument(doc); @@ -135,7 +135,7 @@ public class SimpleAllTests { allEntries.addText("field1", "something moo", 1.0f); allEntries.addText("field2", "else koo", 1.0f); allEntries.reset(); - doc.add(new Field("_all", AllTokenFilter.allTokenStream("_all", allEntries, Lucene.STANDARD_ANALYZER))); + doc.add(new Field("_all", AllTokenStream.allTokenStream("_all", allEntries, Lucene.STANDARD_ANALYZER))); indexWriter.addDocument(doc); @@ -145,7 +145,7 @@ public class SimpleAllTests { allEntries.addText("field1", "else koo", 1.0f); allEntries.addText("field2", "something moo", 1.0f); allEntries.reset(); - doc.add(new Field("_all", AllTokenFilter.allTokenStream("_all", allEntries, Lucene.STANDARD_ANALYZER))); + doc.add(new Field("_all", AllTokenStream.allTokenStream("_all", allEntries, Lucene.STANDARD_ANALYZER))); indexWriter.addDocument(doc); @@ -187,7 +187,7 @@ public class SimpleAllTests { allEntries.addText("field1", "something moo", 1.0f); allEntries.addText("field2", "else koo", 1.0f); allEntries.reset(); - doc.add(new Field("_all", AllTokenFilter.allTokenStream("_all", allEntries, Lucene.STANDARD_ANALYZER))); + doc.add(new Field("_all", AllTokenStream.allTokenStream("_all", allEntries, Lucene.STANDARD_ANALYZER))); indexWriter.addDocument(doc); @@ -197,7 +197,7 @@ public class SimpleAllTests { allEntries.addText("field1", "else koo", 2.0f); allEntries.addText("field2", "something moo", 1.0f); allEntries.reset(); - doc.add(new Field("_all", AllTokenFilter.allTokenStream("_all", allEntries, Lucene.STANDARD_ANALYZER))); + doc.add(new Field("_all", AllTokenStream.allTokenStream("_all", allEntries, Lucene.STANDARD_ANALYZER))); indexWriter.addDocument(doc);