[7.x] Add network from MaxMind Geo ASN database (#61676) (#62898)

This adds the network property from the MaxMind Geo ASN database. 
This enables analysis of IP data based on the subnets that MaxMind have 
previously identified for ASN networks.

closes #60942

Co-authored-by: Peter Ansell <p_ansell@yahoo.com>
This commit is contained in:
Jake Landis 2020-10-01 11:01:44 -05:00 committed by GitHub
parent 95242eccee
commit 0795f4b898
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 18 additions and 7 deletions

View File

@ -40,9 +40,10 @@ and `location`. The fields actually added depend on what has been found and whic
`country_iso_code`, `country_name` and `continent_name`. The fields actually added depend on what has been found and which properties `country_iso_code`, `country_name` and `continent_name`. The fields actually added depend on what has been found and which properties
were configured in `properties`. were configured in `properties`.
* If the GeoLite2 ASN database is used, then the following fields may be added under the `target_field`: `ip`, * If the GeoLite2 ASN database is used, then the following fields may be added under the `target_field`: `ip`,
`asn`, and `organization_name`. The fields actually added depend on what has been found and which properties were configured `asn`, `organization_name` and `network`. The fields actually added depend on what has been found and which properties were configured
in `properties`. in `properties`.
Here is an example that uses the default city database and adds the geographical information to the `geoip` field based on the `ip` field: Here is an example that uses the default city database and adds the geographical information to the `geoip` field based on the `ip` field:
[source,console] [source,console]

View File

@ -19,6 +19,7 @@
package org.elasticsearch.ingest.geoip; package org.elasticsearch.ingest.geoip;
import com.maxmind.db.Network;
import com.maxmind.geoip2.exception.AddressNotFoundException; import com.maxmind.geoip2.exception.AddressNotFoundException;
import com.maxmind.geoip2.model.AsnResponse; import com.maxmind.geoip2.model.AsnResponse;
import com.maxmind.geoip2.model.CityResponse; import com.maxmind.geoip2.model.CityResponse;
@ -345,6 +346,7 @@ public final class GeoIpProcessor extends AbstractProcessor {
Integer asn = response.getAutonomousSystemNumber(); Integer asn = response.getAutonomousSystemNumber();
String organization_name = response.getAutonomousSystemOrganization(); String organization_name = response.getAutonomousSystemOrganization();
Network network = response.getNetwork();
Map<String, Object> geoData = new HashMap<>(); Map<String, Object> geoData = new HashMap<>();
for (Property property : this.properties) { for (Property property : this.properties) {
@ -362,6 +364,11 @@ public final class GeoIpProcessor extends AbstractProcessor {
geoData.put("organization_name", organization_name); geoData.put("organization_name", organization_name);
} }
break; break;
case NETWORK:
if (network != null) {
geoData.put("network", network.toString());
}
break;
} }
} }
return geoData; return geoData;
@ -376,7 +383,7 @@ public final class GeoIpProcessor extends AbstractProcessor {
Property.CONTINENT_NAME, Property.COUNTRY_NAME, Property.COUNTRY_ISO_CODE Property.CONTINENT_NAME, Property.COUNTRY_NAME, Property.COUNTRY_ISO_CODE
)); ));
static final Set<Property> DEFAULT_ASN_PROPERTIES = Collections.unmodifiableSet(EnumSet.of( static final Set<Property> DEFAULT_ASN_PROPERTIES = Collections.unmodifiableSet(EnumSet.of(
Property.IP, Property.ASN, Property.ORGANIZATION_NAME Property.IP, Property.ASN, Property.ORGANIZATION_NAME, Property.NETWORK
)); ));
private final Map<String, DatabaseReaderLazyLoader> databaseReaders; private final Map<String, DatabaseReaderLazyLoader> databaseReaders;
@ -464,7 +471,8 @@ public final class GeoIpProcessor extends AbstractProcessor {
TIMEZONE, TIMEZONE,
LOCATION, LOCATION,
ASN, ASN,
ORGANIZATION_NAME; ORGANIZATION_NAME,
NETWORK;
static final EnumSet<Property> ALL_CITY_PROPERTIES = EnumSet.of( static final EnumSet<Property> ALL_CITY_PROPERTIES = EnumSet.of(
Property.IP, Property.COUNTRY_ISO_CODE, Property.COUNTRY_NAME, Property.CONTINENT_NAME, Property.IP, Property.COUNTRY_ISO_CODE, Property.COUNTRY_NAME, Property.CONTINENT_NAME,
@ -475,7 +483,7 @@ public final class GeoIpProcessor extends AbstractProcessor {
Property.IP, Property.CONTINENT_NAME, Property.COUNTRY_NAME, Property.COUNTRY_ISO_CODE Property.IP, Property.CONTINENT_NAME, Property.COUNTRY_NAME, Property.COUNTRY_ISO_CODE
); );
static final EnumSet<Property> ALL_ASN_PROPERTIES = EnumSet.of( static final EnumSet<Property> ALL_ASN_PROPERTIES = EnumSet.of(
Property.IP, Property.ASN, Property.ORGANIZATION_NAME Property.IP, Property.ASN, Property.ORGANIZATION_NAME, Property.NETWORK
); );
public static Property parseProperty(String databaseType, String value) { public static Property parseProperty(String databaseType, String value) {

View File

@ -189,7 +189,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
config.put("properties", Collections.singletonList(cityProperty)); config.put("properties", Collections.singletonList(cityProperty));
Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config)); Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config));
assertThat(e.getMessage(), equalTo("[properties] illegal property value [" + cityProperty + assertThat(e.getMessage(), equalTo("[properties] illegal property value [" + cityProperty +
"]. valid values are [IP, ASN, ORGANIZATION_NAME]")); "]. valid values are [IP, ASN, ORGANIZATION_NAME, NETWORK]"));
} }
public void testBuildNonExistingDbFile() throws Exception { public void testBuildNonExistingDbFile() throws Exception {

View File

@ -209,10 +209,11 @@ public class GeoIpProcessorTests extends ESTestCase {
assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo(ip)); assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo(ip));
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
Map<String, Object> geoData = (Map<String, Object>) ingestDocument.getSourceAndMetadata().get("target_field"); Map<String, Object> geoData = (Map<String, Object>) ingestDocument.getSourceAndMetadata().get("target_field");
assertThat(geoData.size(), equalTo(3)); assertThat(geoData.size(), equalTo(4));
assertThat(geoData.get("ip"), equalTo(ip)); assertThat(geoData.get("ip"), equalTo(ip));
assertThat(geoData.get("asn"), equalTo(1136)); assertThat(geoData.get("asn"), equalTo(1136));
assertThat(geoData.get("organization_name"), equalTo("KPN B.V.")); assertThat(geoData.get("organization_name"), equalTo("KPN B.V."));
assertThat(geoData.get("network"), equalTo("82.168.0.0/14"));
} }
public void testAddressIsNotInTheDatabase() throws Exception { public void testAddressIsNotInTheDatabase() throws Exception {

View File

@ -304,7 +304,8 @@
index: test index: test
id: 1 id: 1
- match: { _source.field1: "82.171.64.0" } - match: { _source.field1: "82.171.64.0" }
- length: { _source.geoip: 3 } - length: { _source.geoip: 4 }
- match: { _source.geoip.ip: "82.171.64.0" } - match: { _source.geoip.ip: "82.171.64.0" }
- match: { _source.geoip.asn: 1136 } - match: { _source.geoip.asn: 1136 }
- match: { _source.geoip.organization_name: "KPN B.V." } - match: { _source.geoip.organization_name: "KPN B.V." }
- match: { _source.geoip.network: "82.168.0.0/14" }