From 0795f4b898defb6556498bc291fdf352d4d72959 Mon Sep 17 00:00:00 2001 From: Jake Landis Date: Thu, 1 Oct 2020 11:01:44 -0500 Subject: [PATCH] [7.x] Add network from MaxMind Geo ASN database (#61676) (#62898) This adds the network property from the MaxMind Geo ASN database. This enables analysis of IP data based on the subnets that MaxMind have previously identified for ASN networks. closes #60942 Co-authored-by: Peter Ansell --- docs/reference/ingest/processors/geoip.asciidoc | 3 ++- .../elasticsearch/ingest/geoip/GeoIpProcessor.java | 14 +++++++++++--- .../ingest/geoip/GeoIpProcessorFactoryTests.java | 2 +- .../ingest/geoip/GeoIpProcessorTests.java | 3 ++- .../test/ingest_geoip/20_geoip_processor.yml | 3 ++- 5 files changed, 18 insertions(+), 7 deletions(-) diff --git a/docs/reference/ingest/processors/geoip.asciidoc b/docs/reference/ingest/processors/geoip.asciidoc index 2c969543e30..fecf5698bd1 100644 --- a/docs/reference/ingest/processors/geoip.asciidoc +++ b/docs/reference/ingest/processors/geoip.asciidoc @@ -40,9 +40,10 @@ and `location`. The fields actually added depend on what has been found and whic `country_iso_code`, `country_name` and `continent_name`. The fields actually added depend on what has been found and which properties were configured in `properties`. * If the GeoLite2 ASN database is used, then the following fields may be added under the `target_field`: `ip`, -`asn`, and `organization_name`. The fields actually added depend on what has been found and which properties were configured +`asn`, `organization_name` and `network`. The fields actually added depend on what has been found and which properties were configured in `properties`. + Here is an example that uses the default city database and adds the geographical information to the `geoip` field based on the `ip` field: [source,console] diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java index b8b70f6c5cf..0c9080f58af 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java @@ -19,6 +19,7 @@ package org.elasticsearch.ingest.geoip; +import com.maxmind.db.Network; import com.maxmind.geoip2.exception.AddressNotFoundException; import com.maxmind.geoip2.model.AsnResponse; import com.maxmind.geoip2.model.CityResponse; @@ -345,6 +346,7 @@ public final class GeoIpProcessor extends AbstractProcessor { Integer asn = response.getAutonomousSystemNumber(); String organization_name = response.getAutonomousSystemOrganization(); + Network network = response.getNetwork(); Map geoData = new HashMap<>(); for (Property property : this.properties) { @@ -362,6 +364,11 @@ public final class GeoIpProcessor extends AbstractProcessor { geoData.put("organization_name", organization_name); } break; + case NETWORK: + if (network != null) { + geoData.put("network", network.toString()); + } + break; } } return geoData; @@ -376,7 +383,7 @@ public final class GeoIpProcessor extends AbstractProcessor { Property.CONTINENT_NAME, Property.COUNTRY_NAME, Property.COUNTRY_ISO_CODE )); static final Set DEFAULT_ASN_PROPERTIES = Collections.unmodifiableSet(EnumSet.of( - Property.IP, Property.ASN, Property.ORGANIZATION_NAME + Property.IP, Property.ASN, Property.ORGANIZATION_NAME, Property.NETWORK )); private final Map databaseReaders; @@ -464,7 +471,8 @@ public final class GeoIpProcessor extends AbstractProcessor { TIMEZONE, LOCATION, ASN, - ORGANIZATION_NAME; + ORGANIZATION_NAME, + NETWORK; static final EnumSet ALL_CITY_PROPERTIES = EnumSet.of( Property.IP, Property.COUNTRY_ISO_CODE, Property.COUNTRY_NAME, Property.CONTINENT_NAME, @@ -475,7 +483,7 @@ public final class GeoIpProcessor extends AbstractProcessor { Property.IP, Property.CONTINENT_NAME, Property.COUNTRY_NAME, Property.COUNTRY_ISO_CODE ); static final EnumSet ALL_ASN_PROPERTIES = EnumSet.of( - Property.IP, Property.ASN, Property.ORGANIZATION_NAME + Property.IP, Property.ASN, Property.ORGANIZATION_NAME, Property.NETWORK ); public static Property parseProperty(String databaseType, String value) { diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java index b6a6bb97b6f..2b528df3b2e 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java @@ -189,7 +189,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase { config.put("properties", Collections.singletonList(cityProperty)); Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config)); assertThat(e.getMessage(), equalTo("[properties] illegal property value [" + cityProperty + - "]. valid values are [IP, ASN, ORGANIZATION_NAME]")); + "]. valid values are [IP, ASN, ORGANIZATION_NAME, NETWORK]")); } public void testBuildNonExistingDbFile() throws Exception { diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java index c3308089164..e580b1eb4f8 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java @@ -209,10 +209,11 @@ public class GeoIpProcessorTests extends ESTestCase { assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo(ip)); @SuppressWarnings("unchecked") Map geoData = (Map) ingestDocument.getSourceAndMetadata().get("target_field"); - assertThat(geoData.size(), equalTo(3)); + assertThat(geoData.size(), equalTo(4)); assertThat(geoData.get("ip"), equalTo(ip)); assertThat(geoData.get("asn"), equalTo(1136)); assertThat(geoData.get("organization_name"), equalTo("KPN B.V.")); + assertThat(geoData.get("network"), equalTo("82.168.0.0/14")); } public void testAddressIsNotInTheDatabase() throws Exception { diff --git a/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/20_geoip_processor.yml b/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/20_geoip_processor.yml index 15b6d8dc981..3b5ee3bd553 100644 --- a/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/20_geoip_processor.yml +++ b/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/20_geoip_processor.yml @@ -304,7 +304,8 @@ index: test id: 1 - match: { _source.field1: "82.171.64.0" } - - length: { _source.geoip: 3 } + - length: { _source.geoip: 4 } - match: { _source.geoip.ip: "82.171.64.0" } - match: { _source.geoip.asn: 1136 } - match: { _source.geoip.organization_name: "KPN B.V." } + - match: { _source.geoip.network: "82.168.0.0/14" }