honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Made the `term_vector` json field the leading way of configuring term vectors. Supported options: `no`, `yes`, `with_offsets`, `with_positions`, `with_positions_offsets` and`with_positions_offsets_payloads`.

This commit is contained in:
Martijn van Groningen 2013-02-19 20:55:43 +01:00
parent 5b9e9a004a
commit 3b31c1216e
3 changed files with 115 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
src/main/java/org/elasticsearch/index/mapper/core/AbstractFieldMapper.java
View File

@ -615,16 +615,7 @@ public abstract class AbstractFieldMapper<T> implements FieldMapper<T>, Mapper {
builder.field("store", fieldType.stored());
}
if (fieldType.storeTermVectors() != defaultFieldType.storeTermVectors()) {
builder.field("store_term_vector", fieldType.storeTermVectors());
}
if (fieldType.storeTermVectorOffsets() != defaultFieldType.storeTermVectorOffsets()) {
builder.field("store_term_vector_offsets", fieldType.storeTermVectorOffsets());
}
if (fieldType.storeTermVectorPositions() != defaultFieldType.storeTermVectorPositions()) {
builder.field("store_term_vector_positions", fieldType.storeTermVectorPositions());
}
if (fieldType.storeTermVectorPayloads() != defaultFieldType.storeTermVectorPayloads()) {
builder.field("store_term_vector_payloads", fieldType.storeTermVectorPayloads());
builder.field("term_vector", termVectorOptionsToString(fieldType));
}
if (fieldType.omitNorms() != defaultFieldType.omitNorms()) {
builder.field("omit_norms", fieldType.omitNorms());
@ -674,6 +665,28 @@ public abstract class AbstractFieldMapper<T> implements FieldMapper<T>, Mapper {
}
}
protected static String termVectorOptionsToString(FieldType fieldType) {
if (!fieldType.storeTermVectors()) {
return "no";
} else if(!fieldType.storeTermVectorOffsets() && !fieldType.storeTermVectorPositions()) {
return "yes";
} else if (fieldType.storeTermVectorOffsets() && !fieldType.storeTermVectorPositions()) {
return "with_offsets";
} else {
StringBuilder builder = new StringBuilder("with");
if (fieldType.storeTermVectorPositions()) {
builder.append("_positions");
}
if (fieldType.storeTermVectorOffsets()) {
builder.append("_offsets");
}
if (fieldType.storeTermVectorPayloads()) {
builder.append("_payloads");
}
return builder.toString();
}
}
protected static String indexTokenizeOptionToString(boolean indexed, boolean tokenized) {
if (!indexed) {
return "no";

4
src/main/java/org/elasticsearch/index/mapper/core/TypeParsers.java
View File

@ -155,6 +155,10 @@ public class TypeParsers {
} else if ("with_positions_offsets".equals(termVector)) {
builder.storeTermVectorPositions(true);
builder.storeTermVectorOffsets(true);
} else if ("with_positions_offsets_payloads".equals(termVector)) {
builder.storeTermVectorPositions(true);
builder.storeTermVectorOffsets(true);
builder.storeTermVectorPayloads(true);
} else {
throw new MapperParsingException("Wrong value for termVector [" + termVector + "] for field [" + fieldName + "]");
}

88
src/test/java/org/elasticsearch/test/unit/index/mapper/string/SimpleStringMappingTests.java
View File

@ -83,6 +83,10 @@ public class SimpleStringMappingTests {
assertThat(doc.rootDoc().getField("field").fieldType().omitNorms(), equalTo(false));
assertThat(doc.rootDoc().getField("field").fieldType().indexOptions(), equalTo(FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS));
assertThat(doc.rootDoc().getField("field").fieldType().storeTermVectors(), equalTo(false));
assertThat(doc.rootDoc().getField("field").fieldType().storeTermVectorOffsets(), equalTo(false));
assertThat(doc.rootDoc().getField("field").fieldType().storeTermVectorPositions(), equalTo(false));
assertThat(doc.rootDoc().getField("field").fieldType().storeTermVectorPayloads(), equalTo(false));
}
@Test
@ -101,6 +105,10 @@ public class SimpleStringMappingTests {
assertThat(doc.rootDoc().getField("field").fieldType().omitNorms(), equalTo(true));
assertThat(doc.rootDoc().getField("field").fieldType().indexOptions(), equalTo(FieldInfo.IndexOptions.DOCS_ONLY));
assertThat(doc.rootDoc().getField("field").fieldType().storeTermVectors(), equalTo(false));
assertThat(doc.rootDoc().getField("field").fieldType().storeTermVectorOffsets(), equalTo(false));
assertThat(doc.rootDoc().getField("field").fieldType().storeTermVectorPositions(), equalTo(false));
assertThat(doc.rootDoc().getField("field").fieldType().storeTermVectorPayloads(), equalTo(false));
// now test it explicitly set
@ -118,5 +126,85 @@ public class SimpleStringMappingTests {
assertThat(doc.rootDoc().getField("field").fieldType().omitNorms(), equalTo(false));
assertThat(doc.rootDoc().getField("field").fieldType().indexOptions(), equalTo(FieldInfo.IndexOptions.DOCS_AND_FREQS));
assertThat(doc.rootDoc().getField("field").fieldType().storeTermVectors(), equalTo(false));
assertThat(doc.rootDoc().getField("field").fieldType().storeTermVectorOffsets(), equalTo(false));
assertThat(doc.rootDoc().getField("field").fieldType().storeTermVectorPositions(), equalTo(false));
assertThat(doc.rootDoc().getField("field").fieldType().storeTermVectorPayloads(), equalTo(false));
}
@Test
public void testTermVectors() throws Exception {
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("properties")
.startObject("field1")
.field("type", "string")
.field("term_vector", "no")
.endObject()
.startObject("field2")
.field("type", "string")
.field("term_vector", "yes")
.endObject()
.startObject("field3")
.field("type", "string")
.field("term_vector", "with_offsets")
.endObject()
.startObject("field4")
.field("type", "string")
.field("term_vector", "with_positions")
.endObject()
.startObject("field5")
.field("type", "string")
.field("term_vector", "with_positions_offsets")
.endObject()
.startObject("field6")
.field("type", "string")
.field("term_vector", "with_positions_offsets_payloads")
.endObject()
.endObject()
.endObject().endObject().string();
DocumentMapper defaultMapper = MapperTests.newParser().parse(mapping);
ParsedDocument doc = defaultMapper.parse("type", "1", XContentFactory.jsonBuilder()
.startObject()
.field("field1", "1234")
.field("field2", "1234")
.field("field3", "1234")
.field("field4", "1234")
.field("field5", "1234")
.field("field6", "1234")
.endObject()
.bytes());
assertThat(doc.rootDoc().getField("field1").fieldType().storeTermVectors(), equalTo(false));
assertThat(doc.rootDoc().getField("field1").fieldType().storeTermVectorOffsets(), equalTo(false));
assertThat(doc.rootDoc().getField("field1").fieldType().storeTermVectorPositions(), equalTo(false));
assertThat(doc.rootDoc().getField("field1").fieldType().storeTermVectorPayloads(), equalTo(false));
assertThat(doc.rootDoc().getField("field2").fieldType().storeTermVectors(), equalTo(true));
assertThat(doc.rootDoc().getField("field2").fieldType().storeTermVectorOffsets(), equalTo(false));
assertThat(doc.rootDoc().getField("field2").fieldType().storeTermVectorPositions(), equalTo(false));
assertThat(doc.rootDoc().getField("field2").fieldType().storeTermVectorPayloads(), equalTo(false));
assertThat(doc.rootDoc().getField("field3").fieldType().storeTermVectors(), equalTo(true));
assertThat(doc.rootDoc().getField("field3").fieldType().storeTermVectorOffsets(), equalTo(true));
assertThat(doc.rootDoc().getField("field3").fieldType().storeTermVectorPositions(), equalTo(false));
assertThat(doc.rootDoc().getField("field3").fieldType().storeTermVectorPayloads(), equalTo(false));
assertThat(doc.rootDoc().getField("field4").fieldType().storeTermVectors(), equalTo(true));
assertThat(doc.rootDoc().getField("field4").fieldType().storeTermVectorOffsets(), equalTo(false));
assertThat(doc.rootDoc().getField("field4").fieldType().storeTermVectorPositions(), equalTo(true));
assertThat(doc.rootDoc().getField("field4").fieldType().storeTermVectorPayloads(), equalTo(false));
assertThat(doc.rootDoc().getField("field5").fieldType().storeTermVectors(), equalTo(true));
assertThat(doc.rootDoc().getField("field5").fieldType().storeTermVectorOffsets(), equalTo(true));
assertThat(doc.rootDoc().getField("field5").fieldType().storeTermVectorPositions(), equalTo(true));
assertThat(doc.rootDoc().getField("field5").fieldType().storeTermVectorPayloads(), equalTo(false));
assertThat(doc.rootDoc().getField("field6").fieldType().storeTermVectors(), equalTo(true));
assertThat(doc.rootDoc().getField("field6").fieldType().storeTermVectorOffsets(), equalTo(true));
assertThat(doc.rootDoc().getField("field6").fieldType().storeTermVectorPositions(), equalTo(true));
assertThat(doc.rootDoc().getField("field6").fieldType().storeTermVectorPayloads(), equalTo(true));
}
}