Mapper: Ip Type Support (ipv4), auto detection with dynamic mapping, closes #461.
This commit is contained in:
kimchy
parent
6f8b859d90
commit
4579c04a9e
|
|
@ -0,0 +1,279 @@
|
|||
/*
|
||||
* Licensed to Elastic Search and Shay Banon under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. Elastic Search licenses this
|
||||
* file to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.index.mapper.xcontent;
|
||||
|
||||
import org.apache.lucene.analysis.NumericTokenStream;
|
||||
import org.apache.lucene.document.Field;
|
||||
import org.apache.lucene.document.Fieldable;
|
||||
import org.apache.lucene.search.Filter;
|
||||
import org.apache.lucene.search.NumericRangeFilter;
|
||||
import org.apache.lucene.search.NumericRangeQuery;
|
||||
import org.apache.lucene.search.Query;
|
||||
import org.apache.lucene.util.NumericUtils;
|
||||
import org.elasticsearch.ElasticSearchIllegalArgumentException;
|
||||
import org.elasticsearch.common.Numbers;
|
||||
import org.elasticsearch.common.Strings;
|
||||
import org.elasticsearch.common.xcontent.XContentBuilder;
|
||||
import org.elasticsearch.common.xcontent.XContentParser;
|
||||
import org.elasticsearch.index.analysis.NamedAnalyzer;
|
||||
import org.elasticsearch.index.analysis.NumericAnalyzer;
|
||||
import org.elasticsearch.index.analysis.NumericTokenizer;
|
||||
import org.elasticsearch.index.field.data.FieldDataType;
|
||||
import org.elasticsearch.index.mapper.MapperParsingException;
|
||||
import org.elasticsearch.index.mapper.MergeMappingException;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.Reader;
|
||||
import java.util.Map;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import static org.elasticsearch.index.mapper.xcontent.XContentMapperBuilders.*;
|
||||
import static org.elasticsearch.index.mapper.xcontent.XContentTypeParsers.*;
|
||||
|
||||
/**
|
||||
* @author kimchy (shay.banon)
|
||||
*/
|
||||
public class IpFieldMapper extends NumberFieldMapper<Long> {
|
||||
|
||||
public static final String CONTENT_TYPE = "ip";
|
||||
|
||||
public static String longToIp(long longIp) {
|
||||
int octet3 = (int) ((longIp >> 24) % 256);
|
||||
int octet2 = (int) ((longIp >> 16) % 256);
|
||||
int octet1 = (int) ((longIp >> 8) % 256);
|
||||
int octet0 = (int) ((longIp) % 256);
|
||||
return octet3 + "." + octet2 + "." + octet1 + "." + octet0;
|
||||
}
|
||||
|
||||
private static final Pattern pattern = Pattern.compile("\\.");
|
||||
|
||||
public static long ipToLong(String ip) throws ElasticSearchIllegalArgumentException {
|
||||
try {
|
||||
String[] octets = pattern.split(ip);
|
||||
if (octets.length != 4) {
|
||||
throw new ElasticSearchIllegalArgumentException("failed ot parse ip [" + ip + "], not full ip address (4 dots)");
|
||||
}
|
||||
return (Long.parseLong(octets[0]) << 24) + (Integer.parseInt(octets[1]) << 16) +
|
||||
(Integer.parseInt(octets[2]) << 8) + Integer.parseInt(octets[3]);
|
||||
} catch (Exception e) {
|
||||
if (e instanceof ElasticSearchIllegalArgumentException) {
|
||||
throw (ElasticSearchIllegalArgumentException) e;
|
||||
}
|
||||
throw new ElasticSearchIllegalArgumentException("failed to parse ip [" + ip + "]", e);
|
||||
}
|
||||
}
|
||||
|
||||
public static class Defaults extends NumberFieldMapper.Defaults {
|
||||
public static final String NULL_VALUE = null;
|
||||
}
|
||||
|
||||
public static class Builder extends NumberFieldMapper.Builder<Builder, IpFieldMapper> {
|
||||
|
||||
protected String nullValue = Defaults.NULL_VALUE;
|
||||
|
||||
public Builder(String name) {
|
||||
super(name);
|
||||
builder = this;
|
||||
}
|
||||
|
||||
public Builder nullValue(String nullValue) {
|
||||
this.nullValue = nullValue;
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override public IpFieldMapper build(BuilderContext context) {
|
||||
IpFieldMapper fieldMapper = new IpFieldMapper(buildNames(context),
|
||||
precisionStep, index, store, boost, omitNorms, omitTermFreqAndPositions, nullValue);
|
||||
fieldMapper.includeInAll(includeInAll);
|
||||
return fieldMapper;
|
||||
}
|
||||
}
|
||||
|
||||
public static class TypeParser implements XContentMapper.TypeParser {
|
||||
@Override public XContentMapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
|
||||
IpFieldMapper.Builder builder = ipField(name);
|
||||
parseNumberField(builder, name, node, parserContext);
|
||||
for (Map.Entry<String, Object> entry : node.entrySet()) {
|
||||
String propName = Strings.toUnderscoreCase(entry.getKey());
|
||||
Object propNode = entry.getValue();
|
||||
if (propName.equals("null_value")) {
|
||||
builder.nullValue(propNode.toString());
|
||||
}
|
||||
}
|
||||
return builder;
|
||||
}
|
||||
}
|
||||
|
||||
private String nullValue;
|
||||
|
||||
protected IpFieldMapper(Names names, int precisionStep,
|
||||
Field.Index index, Field.Store store,
|
||||
float boost, boolean omitNorms, boolean omitTermFreqAndPositions,
|
||||
String nullValue) {
|
||||
super(names, precisionStep, index, store, boost, omitNorms, omitTermFreqAndPositions,
|
||||
new NamedAnalyzer("_ip/" + precisionStep, new NumericIpAnalyzer(precisionStep)),
|
||||
new NamedAnalyzer("_ip/max", new NumericIpAnalyzer(Integer.MAX_VALUE)));
|
||||
this.nullValue = nullValue;
|
||||
}
|
||||
|
||||
@Override protected int maxPrecisionStep() {
|
||||
return 64;
|
||||
}
|
||||
|
||||
@Override public Long value(Fieldable field) {
|
||||
byte[] value = field.getBinaryValue();
|
||||
if (value == null) {
|
||||
return null;
|
||||
}
|
||||
return Numbers.bytesToLong(value);
|
||||
}
|
||||
|
||||
@Override public Long valueFromString(String value) {
|
||||
return ipToLong(value);
|
||||
}
|
||||
|
||||
/**
|
||||
* Dates should return as a string, delegates to {@link #valueAsString(org.apache.lucene.document.Fieldable)}.
|
||||
*/
|
||||
@Override public Object valueForSearch(Fieldable field) {
|
||||
return valueAsString(field);
|
||||
}
|
||||
|
||||
@Override public String valueAsString(Fieldable field) {
|
||||
Long value = value(field);
|
||||
if (value == null) {
|
||||
return null;
|
||||
}
|
||||
return longToIp(value);
|
||||
}
|
||||
|
||||
@Override public String indexedValue(String value) {
|
||||
return NumericUtils.longToPrefixCoded(ipToLong(value));
|
||||
}
|
||||
|
||||
@Override public Query rangeQuery(String lowerTerm, String upperTerm, boolean includeLower, boolean includeUpper) {
|
||||
return NumericRangeQuery.newLongRange(names.indexName(), precisionStep,
|
||||
lowerTerm == null ? null : ipToLong(lowerTerm),
|
||||
upperTerm == null ? null : ipToLong(upperTerm),
|
||||
includeLower, includeUpper);
|
||||
}
|
||||
|
||||
@Override public Filter rangeFilter(String lowerTerm, String upperTerm, boolean includeLower, boolean includeUpper) {
|
||||
return NumericRangeFilter.newLongRange(names.indexName(), precisionStep,
|
||||
lowerTerm == null ? null : ipToLong(lowerTerm),
|
||||
upperTerm == null ? null : ipToLong(upperTerm),
|
||||
includeLower, includeUpper);
|
||||
}
|
||||
|
||||
@Override protected Field parseCreateField(ParseContext context) throws IOException {
|
||||
String ipAsString;
|
||||
if (context.externalValueSet()) {
|
||||
ipAsString = (String) context.externalValue();
|
||||
if (ipAsString == null) {
|
||||
ipAsString = nullValue;
|
||||
}
|
||||
} else {
|
||||
if (context.parser().currentToken() == XContentParser.Token.VALUE_NULL) {
|
||||
ipAsString = nullValue;
|
||||
} else {
|
||||
ipAsString = context.parser().text();
|
||||
}
|
||||
}
|
||||
|
||||
if (ipAsString == null) {
|
||||
return null;
|
||||
}
|
||||
if (includeInAll == null || includeInAll) {
|
||||
context.allEntries().addText(names.fullName(), ipAsString, boost);
|
||||
}
|
||||
|
||||
long value = ipToLong(ipAsString);
|
||||
Field field = null;
|
||||
if (stored()) {
|
||||
field = new Field(names.indexName(), Numbers.longToBytes(value), store);
|
||||
if (indexed()) {
|
||||
field.setTokenStream(popCachedStream(precisionStep).setLongValue(value));
|
||||
}
|
||||
} else if (indexed()) {
|
||||
field = new Field(names.indexName(), popCachedStream(precisionStep).setLongValue(value));
|
||||
}
|
||||
return field;
|
||||
}
|
||||
|
||||
@Override public FieldDataType fieldDataType() {
|
||||
return FieldDataType.DefaultTypes.LONG;
|
||||
}
|
||||
|
||||
@Override protected String contentType() {
|
||||
return CONTENT_TYPE;
|
||||
}
|
||||
|
||||
@Override public void merge(XContentMapper mergeWith, MergeContext mergeContext) throws MergeMappingException {
|
||||
super.merge(mergeWith, mergeContext);
|
||||
if (!this.getClass().equals(mergeWith.getClass())) {
|
||||
return;
|
||||
}
|
||||
if (!mergeContext.mergeFlags().simulate()) {
|
||||
this.nullValue = ((IpFieldMapper) mergeWith).nullValue;
|
||||
}
|
||||
}
|
||||
|
||||
@Override protected void doXContentBody(XContentBuilder builder) throws IOException {
|
||||
super.doXContentBody(builder);
|
||||
if (nullValue != null) {
|
||||
builder.field("null_value", nullValue);
|
||||
}
|
||||
if (includeInAll != null) {
|
||||
builder.field("include_in_all", includeInAll);
|
||||
}
|
||||
}
|
||||
|
||||
public static class NumericIpAnalyzer extends NumericAnalyzer<NumericIpTokenizer> {
|
||||
|
||||
private final int precisionStep;
|
||||
|
||||
public NumericIpAnalyzer() {
|
||||
this(NumericUtils.PRECISION_STEP_DEFAULT);
|
||||
}
|
||||
|
||||
public NumericIpAnalyzer(int precisionStep) {
|
||||
this.precisionStep = precisionStep;
|
||||
}
|
||||
|
||||
@Override protected NumericIpTokenizer createNumericTokenizer(Reader reader, char[] buffer) throws IOException {
|
||||
return new NumericIpTokenizer(reader, precisionStep, buffer);
|
||||
}
|
||||
}
|
||||
|
||||
public static class NumericIpTokenizer extends NumericTokenizer {
|
||||
|
||||
public NumericIpTokenizer(Reader reader, int precisionStep) throws IOException {
|
||||
super(reader, new NumericTokenStream(precisionStep), null);
|
||||
}
|
||||
|
||||
public NumericIpTokenizer(Reader reader, int precisionStep, char[] buffer) throws IOException {
|
||||
super(reader, new NumericTokenStream(precisionStep), buffer, null);
|
||||
}
|
||||
|
||||
@Override protected void setValue(NumericTokenStream tokenStream, String value) {
|
||||
tokenStream.setLongValue(ipToLong(value));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -386,7 +386,7 @@ public class ObjectMapper implements XContentMapper, IncludeInAllMapper {
|
|||
if (token == XContentParser.Token.VALUE_STRING) {
|
||||
String text = context.parser().text();
|
||||
// check if it fits one of the date formats
|
||||
boolean isDate = false;
|
||||
boolean resolved = false;
|
||||
// a safe check since "1" gets parsed as well
|
||||
if (text.contains(":") || text.contains("-") || text.contains("/")) {
|
||||
for (FormatDateTimeFormatter dateTimeFormatter : context.root().dateTimeFormatters()) {
|
||||
|
|
@ -397,14 +397,28 @@ public class ObjectMapper implements XContentMapper, IncludeInAllMapper {
|
|||
builder = dateField(currentFieldName).dateTimeFormatter(dateTimeFormatter);
|
||||
}
|
||||
mapper = builder.build(builderContext);
|
||||
isDate = true;
|
||||
resolved = true;
|
||||
break;
|
||||
} catch (Exception e) {
|
||||
// failure to parse this, continue
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!isDate) {
|
||||
// check if its an ip
|
||||
if (!resolved && text.indexOf('.') != -1) {
|
||||
try {
|
||||
IpFieldMapper.ipToLong(text);
|
||||
XContentMapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "ip");
|
||||
if (builder == null) {
|
||||
builder = ipField(currentFieldName);
|
||||
}
|
||||
mapper = builder.build(builderContext);
|
||||
resolved = true;
|
||||
} catch (Exception e) {
|
||||
// failure to parse, not ip...
|
||||
}
|
||||
}
|
||||
if (!resolved) {
|
||||
XContentMapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "string");
|
||||
if (builder == null) {
|
||||
builder = stringField(currentFieldName);
|
||||
|
|
|
|||
|
|
@ -75,6 +75,7 @@ public class XContentDocumentMapperParser extends AbstractIndexComponent impleme
|
|||
.put(BooleanFieldMapper.CONTENT_TYPE, new BooleanFieldMapper.TypeParser())
|
||||
.put(BinaryFieldMapper.CONTENT_TYPE, new BinaryFieldMapper.TypeParser())
|
||||
.put(DateFieldMapper.CONTENT_TYPE, new DateFieldMapper.TypeParser())
|
||||
.put(IpFieldMapper.CONTENT_TYPE, new IpFieldMapper.TypeParser())
|
||||
.put(StringFieldMapper.CONTENT_TYPE, new StringFieldMapper.TypeParser())
|
||||
.put(ObjectMapper.CONTENT_TYPE, new ObjectMapper.TypeParser())
|
||||
.put(MultiFieldMapper.CONTENT_TYPE, new MultiFieldMapper.TypeParser())
|
||||
|
|
|
|||
|
|
@ -88,6 +88,10 @@ public final class XContentMapperBuilders {
|
|||
return new DateFieldMapper.Builder(name);
|
||||
}
|
||||
|
||||
public static IpFieldMapper.Builder ipField(String name) {
|
||||
return new IpFieldMapper.Builder(name);
|
||||
}
|
||||
|
||||
public static ShortFieldMapper.Builder shortField(String name) {
|
||||
return new ShortFieldMapper.Builder(name);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,56 @@
|
|||
/*
|
||||
* Licensed to Elastic Search and Shay Banon under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. Elastic Search licenses this
|
||||
* file to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.index.mapper.xcontent.ip;
|
||||
|
||||
import org.elasticsearch.common.xcontent.XContentFactory;
|
||||
import org.elasticsearch.index.mapper.ParsedDocument;
|
||||
import org.elasticsearch.index.mapper.xcontent.MapperTests;
|
||||
import org.elasticsearch.index.mapper.xcontent.XContentDocumentMapper;
|
||||
import org.testng.annotations.Test;
|
||||
|
||||
import static org.hamcrest.MatcherAssert.*;
|
||||
import static org.hamcrest.Matchers.*;
|
||||
|
||||
/**
|
||||
* @author kimchy (shay.banon)
|
||||
*/
|
||||
public class SimpleIpMappingTests {
|
||||
|
||||
@Test public void testAutoIpDetection() throws Exception {
|
||||
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
|
||||
.startObject("properties").endObject()
|
||||
.endObject().endObject().string();
|
||||
|
||||
XContentDocumentMapper defaultMapper = MapperTests.newParser().parse(mapping);
|
||||
|
||||
ParsedDocument doc = defaultMapper.parse("type", "1", XContentFactory.jsonBuilder()
|
||||
.startObject()
|
||||
.field("ip1", "127.0.0.1")
|
||||
.field("ip2", "0.1")
|
||||
.field("ip3", "127.0.0.1.2")
|
||||
.endObject()
|
||||
.copiedBytes());
|
||||
|
||||
assertThat(doc.doc().getField("ip1"), notNullValue());
|
||||
assertThat(doc.doc().get("ip1"), nullValue()); // its numeric
|
||||
assertThat(doc.doc().get("ip2"), equalTo("0.1"));
|
||||
assertThat(doc.doc().get("ip3"), equalTo("127.0.0.1.2"));
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,72 @@
|
|||
/*
|
||||
* Licensed to Elastic Search and Shay Banon under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. Elastic Search licenses this
|
||||
* file to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.test.integration.search.ip;
|
||||
|
||||
import org.elasticsearch.action.search.SearchResponse;
|
||||
import org.elasticsearch.client.Client;
|
||||
import org.elasticsearch.common.settings.ImmutableSettings;
|
||||
import org.elasticsearch.test.integration.AbstractNodesTests;
|
||||
import org.testng.annotations.AfterClass;
|
||||
import org.testng.annotations.BeforeClass;
|
||||
import org.testng.annotations.Test;
|
||||
|
||||
import static org.elasticsearch.index.query.xcontent.QueryBuilders.*;
|
||||
import static org.hamcrest.MatcherAssert.*;
|
||||
import static org.hamcrest.Matchers.*;
|
||||
|
||||
/**
|
||||
* @author kimchy (shay.banon)
|
||||
*/
|
||||
public class IpSearchTests extends AbstractNodesTests {
|
||||
|
||||
private Client client;
|
||||
|
||||
@BeforeClass public void createNodes() throws Exception {
|
||||
startNode("node1");
|
||||
client = getClient();
|
||||
}
|
||||
|
||||
@AfterClass public void closeNodes() {
|
||||
client.close();
|
||||
closeAllNodes();
|
||||
}
|
||||
|
||||
protected Client getClient() {
|
||||
return client("node1");
|
||||
}
|
||||
|
||||
@Test public void filterExistsMissingTests() throws Exception {
|
||||
try {
|
||||
client.admin().indices().prepareDelete("test").execute().actionGet();
|
||||
} catch (Exception e) {
|
||||
// ignore
|
||||
}
|
||||
|
||||
client.admin().indices().prepareCreate("test").setSettings(ImmutableSettings.settingsBuilder().put("number_of_shards", 1)).execute().actionGet();
|
||||
|
||||
client.prepareIndex("test", "type1", "1").setSource("from", "192.168.0.5", "to", "192.168.0.10").setRefresh(true).execute().actionGet();
|
||||
|
||||
SearchResponse search = client.prepareSearch()
|
||||
.setQuery(boolQuery().must(rangeQuery("from").lt("192.168.0.7")).must(rangeQuery("to").gt("192.168.0.7")))
|
||||
.execute().actionGet();
|
||||
|
||||
assertThat(search.hits().totalHits(), equalTo(1l));
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue