From 4579c04a9ea4a167a6ed87ce676d16736089f729 Mon Sep 17 00:00:00 2001 From: kimchy Date: Thu, 28 Oct 2010 12:02:20 +0200 Subject: [PATCH] Mapper: Ip Type Support (ipv4), auto detection with dynamic mapping, closes #461. --- .../index/mapper/xcontent/IpFieldMapper.java | 279 ++++++++++++++++++ .../index/mapper/xcontent/ObjectMapper.java | 20 +- .../XContentDocumentMapperParser.java | 1 + .../xcontent/XContentMapperBuilders.java | 4 + .../xcontent/ip/SimpleIpMappingTests.java | 56 ++++ .../integration/search/ip/IpSearchTests.java | 72 +++++ 6 files changed, 429 insertions(+), 3 deletions(-) create mode 100644 modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/IpFieldMapper.java create mode 100644 modules/elasticsearch/src/test/java/org/elasticsearch/index/mapper/xcontent/ip/SimpleIpMappingTests.java create mode 100644 modules/test/integration/src/test/java/org/elasticsearch/test/integration/search/ip/IpSearchTests.java diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/IpFieldMapper.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/IpFieldMapper.java new file mode 100644 index 00000000000..92b35278b44 --- /dev/null +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/IpFieldMapper.java @@ -0,0 +1,279 @@ +/* + * Licensed to Elastic Search and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Elastic Search licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.mapper.xcontent; + +import org.apache.lucene.analysis.NumericTokenStream; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.Fieldable; +import org.apache.lucene.search.Filter; +import org.apache.lucene.search.NumericRangeFilter; +import org.apache.lucene.search.NumericRangeQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.util.NumericUtils; +import org.elasticsearch.ElasticSearchIllegalArgumentException; +import org.elasticsearch.common.Numbers; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.analysis.NamedAnalyzer; +import org.elasticsearch.index.analysis.NumericAnalyzer; +import org.elasticsearch.index.analysis.NumericTokenizer; +import org.elasticsearch.index.field.data.FieldDataType; +import org.elasticsearch.index.mapper.MapperParsingException; +import org.elasticsearch.index.mapper.MergeMappingException; + +import java.io.IOException; +import java.io.Reader; +import java.util.Map; +import java.util.regex.Pattern; + +import static org.elasticsearch.index.mapper.xcontent.XContentMapperBuilders.*; +import static org.elasticsearch.index.mapper.xcontent.XContentTypeParsers.*; + +/** + * @author kimchy (shay.banon) + */ +public class IpFieldMapper extends NumberFieldMapper { + + public static final String CONTENT_TYPE = "ip"; + + public static String longToIp(long longIp) { + int octet3 = (int) ((longIp >> 24) % 256); + int octet2 = (int) ((longIp >> 16) % 256); + int octet1 = (int) ((longIp >> 8) % 256); + int octet0 = (int) ((longIp) % 256); + return octet3 + "." + octet2 + "." + octet1 + "." + octet0; + } + + private static final Pattern pattern = Pattern.compile("\\."); + + public static long ipToLong(String ip) throws ElasticSearchIllegalArgumentException { + try { + String[] octets = pattern.split(ip); + if (octets.length != 4) { + throw new ElasticSearchIllegalArgumentException("failed ot parse ip [" + ip + "], not full ip address (4 dots)"); + } + return (Long.parseLong(octets[0]) << 24) + (Integer.parseInt(octets[1]) << 16) + + (Integer.parseInt(octets[2]) << 8) + Integer.parseInt(octets[3]); + } catch (Exception e) { + if (e instanceof ElasticSearchIllegalArgumentException) { + throw (ElasticSearchIllegalArgumentException) e; + } + throw new ElasticSearchIllegalArgumentException("failed to parse ip [" + ip + "]", e); + } + } + + public static class Defaults extends NumberFieldMapper.Defaults { + public static final String NULL_VALUE = null; + } + + public static class Builder extends NumberFieldMapper.Builder { + + protected String nullValue = Defaults.NULL_VALUE; + + public Builder(String name) { + super(name); + builder = this; + } + + public Builder nullValue(String nullValue) { + this.nullValue = nullValue; + return this; + } + + @Override public IpFieldMapper build(BuilderContext context) { + IpFieldMapper fieldMapper = new IpFieldMapper(buildNames(context), + precisionStep, index, store, boost, omitNorms, omitTermFreqAndPositions, nullValue); + fieldMapper.includeInAll(includeInAll); + return fieldMapper; + } + } + + public static class TypeParser implements XContentMapper.TypeParser { + @Override public XContentMapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { + IpFieldMapper.Builder builder = ipField(name); + parseNumberField(builder, name, node, parserContext); + for (Map.Entry entry : node.entrySet()) { + String propName = Strings.toUnderscoreCase(entry.getKey()); + Object propNode = entry.getValue(); + if (propName.equals("null_value")) { + builder.nullValue(propNode.toString()); + } + } + return builder; + } + } + + private String nullValue; + + protected IpFieldMapper(Names names, int precisionStep, + Field.Index index, Field.Store store, + float boost, boolean omitNorms, boolean omitTermFreqAndPositions, + String nullValue) { + super(names, precisionStep, index, store, boost, omitNorms, omitTermFreqAndPositions, + new NamedAnalyzer("_ip/" + precisionStep, new NumericIpAnalyzer(precisionStep)), + new NamedAnalyzer("_ip/max", new NumericIpAnalyzer(Integer.MAX_VALUE))); + this.nullValue = nullValue; + } + + @Override protected int maxPrecisionStep() { + return 64; + } + + @Override public Long value(Fieldable field) { + byte[] value = field.getBinaryValue(); + if (value == null) { + return null; + } + return Numbers.bytesToLong(value); + } + + @Override public Long valueFromString(String value) { + return ipToLong(value); + } + + /** + * Dates should return as a string, delegates to {@link #valueAsString(org.apache.lucene.document.Fieldable)}. + */ + @Override public Object valueForSearch(Fieldable field) { + return valueAsString(field); + } + + @Override public String valueAsString(Fieldable field) { + Long value = value(field); + if (value == null) { + return null; + } + return longToIp(value); + } + + @Override public String indexedValue(String value) { + return NumericUtils.longToPrefixCoded(ipToLong(value)); + } + + @Override public Query rangeQuery(String lowerTerm, String upperTerm, boolean includeLower, boolean includeUpper) { + return NumericRangeQuery.newLongRange(names.indexName(), precisionStep, + lowerTerm == null ? null : ipToLong(lowerTerm), + upperTerm == null ? null : ipToLong(upperTerm), + includeLower, includeUpper); + } + + @Override public Filter rangeFilter(String lowerTerm, String upperTerm, boolean includeLower, boolean includeUpper) { + return NumericRangeFilter.newLongRange(names.indexName(), precisionStep, + lowerTerm == null ? null : ipToLong(lowerTerm), + upperTerm == null ? null : ipToLong(upperTerm), + includeLower, includeUpper); + } + + @Override protected Field parseCreateField(ParseContext context) throws IOException { + String ipAsString; + if (context.externalValueSet()) { + ipAsString = (String) context.externalValue(); + if (ipAsString == null) { + ipAsString = nullValue; + } + } else { + if (context.parser().currentToken() == XContentParser.Token.VALUE_NULL) { + ipAsString = nullValue; + } else { + ipAsString = context.parser().text(); + } + } + + if (ipAsString == null) { + return null; + } + if (includeInAll == null || includeInAll) { + context.allEntries().addText(names.fullName(), ipAsString, boost); + } + + long value = ipToLong(ipAsString); + Field field = null; + if (stored()) { + field = new Field(names.indexName(), Numbers.longToBytes(value), store); + if (indexed()) { + field.setTokenStream(popCachedStream(precisionStep).setLongValue(value)); + } + } else if (indexed()) { + field = new Field(names.indexName(), popCachedStream(precisionStep).setLongValue(value)); + } + return field; + } + + @Override public FieldDataType fieldDataType() { + return FieldDataType.DefaultTypes.LONG; + } + + @Override protected String contentType() { + return CONTENT_TYPE; + } + + @Override public void merge(XContentMapper mergeWith, MergeContext mergeContext) throws MergeMappingException { + super.merge(mergeWith, mergeContext); + if (!this.getClass().equals(mergeWith.getClass())) { + return; + } + if (!mergeContext.mergeFlags().simulate()) { + this.nullValue = ((IpFieldMapper) mergeWith).nullValue; + } + } + + @Override protected void doXContentBody(XContentBuilder builder) throws IOException { + super.doXContentBody(builder); + if (nullValue != null) { + builder.field("null_value", nullValue); + } + if (includeInAll != null) { + builder.field("include_in_all", includeInAll); + } + } + + public static class NumericIpAnalyzer extends NumericAnalyzer { + + private final int precisionStep; + + public NumericIpAnalyzer() { + this(NumericUtils.PRECISION_STEP_DEFAULT); + } + + public NumericIpAnalyzer(int precisionStep) { + this.precisionStep = precisionStep; + } + + @Override protected NumericIpTokenizer createNumericTokenizer(Reader reader, char[] buffer) throws IOException { + return new NumericIpTokenizer(reader, precisionStep, buffer); + } + } + + public static class NumericIpTokenizer extends NumericTokenizer { + + public NumericIpTokenizer(Reader reader, int precisionStep) throws IOException { + super(reader, new NumericTokenStream(precisionStep), null); + } + + public NumericIpTokenizer(Reader reader, int precisionStep, char[] buffer) throws IOException { + super(reader, new NumericTokenStream(precisionStep), buffer, null); + } + + @Override protected void setValue(NumericTokenStream tokenStream, String value) { + tokenStream.setLongValue(ipToLong(value)); + } + } +} \ No newline at end of file diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/ObjectMapper.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/ObjectMapper.java index 5e2c115bc0f..c159c7ec2f5 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/ObjectMapper.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/ObjectMapper.java @@ -386,7 +386,7 @@ public class ObjectMapper implements XContentMapper, IncludeInAllMapper { if (token == XContentParser.Token.VALUE_STRING) { String text = context.parser().text(); // check if it fits one of the date formats - boolean isDate = false; + boolean resolved = false; // a safe check since "1" gets parsed as well if (text.contains(":") || text.contains("-") || text.contains("/")) { for (FormatDateTimeFormatter dateTimeFormatter : context.root().dateTimeFormatters()) { @@ -397,14 +397,28 @@ public class ObjectMapper implements XContentMapper, IncludeInAllMapper { builder = dateField(currentFieldName).dateTimeFormatter(dateTimeFormatter); } mapper = builder.build(builderContext); - isDate = true; + resolved = true; break; } catch (Exception e) { // failure to parse this, continue } } } - if (!isDate) { + // check if its an ip + if (!resolved && text.indexOf('.') != -1) { + try { + IpFieldMapper.ipToLong(text); + XContentMapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "ip"); + if (builder == null) { + builder = ipField(currentFieldName); + } + mapper = builder.build(builderContext); + resolved = true; + } catch (Exception e) { + // failure to parse, not ip... + } + } + if (!resolved) { XContentMapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "string"); if (builder == null) { builder = stringField(currentFieldName); diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/XContentDocumentMapperParser.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/XContentDocumentMapperParser.java index e06e188c938..08537c12177 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/XContentDocumentMapperParser.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/XContentDocumentMapperParser.java @@ -75,6 +75,7 @@ public class XContentDocumentMapperParser extends AbstractIndexComponent impleme .put(BooleanFieldMapper.CONTENT_TYPE, new BooleanFieldMapper.TypeParser()) .put(BinaryFieldMapper.CONTENT_TYPE, new BinaryFieldMapper.TypeParser()) .put(DateFieldMapper.CONTENT_TYPE, new DateFieldMapper.TypeParser()) + .put(IpFieldMapper.CONTENT_TYPE, new IpFieldMapper.TypeParser()) .put(StringFieldMapper.CONTENT_TYPE, new StringFieldMapper.TypeParser()) .put(ObjectMapper.CONTENT_TYPE, new ObjectMapper.TypeParser()) .put(MultiFieldMapper.CONTENT_TYPE, new MultiFieldMapper.TypeParser()) diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/XContentMapperBuilders.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/XContentMapperBuilders.java index 045ac6608ff..ab2d89ff554 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/XContentMapperBuilders.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/xcontent/XContentMapperBuilders.java @@ -88,6 +88,10 @@ public final class XContentMapperBuilders { return new DateFieldMapper.Builder(name); } + public static IpFieldMapper.Builder ipField(String name) { + return new IpFieldMapper.Builder(name); + } + public static ShortFieldMapper.Builder shortField(String name) { return new ShortFieldMapper.Builder(name); } diff --git a/modules/elasticsearch/src/test/java/org/elasticsearch/index/mapper/xcontent/ip/SimpleIpMappingTests.java b/modules/elasticsearch/src/test/java/org/elasticsearch/index/mapper/xcontent/ip/SimpleIpMappingTests.java new file mode 100644 index 00000000000..e5d94ea88c3 --- /dev/null +++ b/modules/elasticsearch/src/test/java/org/elasticsearch/index/mapper/xcontent/ip/SimpleIpMappingTests.java @@ -0,0 +1,56 @@ +/* + * Licensed to Elastic Search and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Elastic Search licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.mapper.xcontent.ip; + +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.index.mapper.ParsedDocument; +import org.elasticsearch.index.mapper.xcontent.MapperTests; +import org.elasticsearch.index.mapper.xcontent.XContentDocumentMapper; +import org.testng.annotations.Test; + +import static org.hamcrest.MatcherAssert.*; +import static org.hamcrest.Matchers.*; + +/** + * @author kimchy (shay.banon) + */ +public class SimpleIpMappingTests { + + @Test public void testAutoIpDetection() throws Exception { + String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") + .startObject("properties").endObject() + .endObject().endObject().string(); + + XContentDocumentMapper defaultMapper = MapperTests.newParser().parse(mapping); + + ParsedDocument doc = defaultMapper.parse("type", "1", XContentFactory.jsonBuilder() + .startObject() + .field("ip1", "127.0.0.1") + .field("ip2", "0.1") + .field("ip3", "127.0.0.1.2") + .endObject() + .copiedBytes()); + + assertThat(doc.doc().getField("ip1"), notNullValue()); + assertThat(doc.doc().get("ip1"), nullValue()); // its numeric + assertThat(doc.doc().get("ip2"), equalTo("0.1")); + assertThat(doc.doc().get("ip3"), equalTo("127.0.0.1.2")); + } +} diff --git a/modules/test/integration/src/test/java/org/elasticsearch/test/integration/search/ip/IpSearchTests.java b/modules/test/integration/src/test/java/org/elasticsearch/test/integration/search/ip/IpSearchTests.java new file mode 100644 index 00000000000..a5f07c02fb3 --- /dev/null +++ b/modules/test/integration/src/test/java/org/elasticsearch/test/integration/search/ip/IpSearchTests.java @@ -0,0 +1,72 @@ +/* + * Licensed to Elastic Search and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Elastic Search licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.test.integration.search.ip; + +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.client.Client; +import org.elasticsearch.common.settings.ImmutableSettings; +import org.elasticsearch.test.integration.AbstractNodesTests; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import static org.elasticsearch.index.query.xcontent.QueryBuilders.*; +import static org.hamcrest.MatcherAssert.*; +import static org.hamcrest.Matchers.*; + +/** + * @author kimchy (shay.banon) + */ +public class IpSearchTests extends AbstractNodesTests { + + private Client client; + + @BeforeClass public void createNodes() throws Exception { + startNode("node1"); + client = getClient(); + } + + @AfterClass public void closeNodes() { + client.close(); + closeAllNodes(); + } + + protected Client getClient() { + return client("node1"); + } + + @Test public void filterExistsMissingTests() throws Exception { + try { + client.admin().indices().prepareDelete("test").execute().actionGet(); + } catch (Exception e) { + // ignore + } + + client.admin().indices().prepareCreate("test").setSettings(ImmutableSettings.settingsBuilder().put("number_of_shards", 1)).execute().actionGet(); + + client.prepareIndex("test", "type1", "1").setSource("from", "192.168.0.5", "to", "192.168.0.10").setRefresh(true).execute().actionGet(); + + SearchResponse search = client.prepareSearch() + .setQuery(boolQuery().must(rangeQuery("from").lt("192.168.0.7")).must(rangeQuery("to").gt("192.168.0.7"))) + .execute().actionGet(); + + assertThat(search.hits().totalHits(), equalTo(1l)); + } +}