[7.x] Add network from MaxMind Geo ASN database (#61676) (#62898)

This adds the network property from the MaxMind Geo ASN database. 
This enables analysis of IP data based on the subnets that MaxMind have 
previously identified for ASN networks.

closes #60942

Co-authored-by: Peter Ansell <p_ansell@yahoo.com>
This commit is contained in:
Jake Landis 2020-10-01 11:01:44 -05:00 committed by GitHub
parent 95242eccee
commit 0795f4b898
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 18 additions and 7 deletions

View File

@ -40,9 +40,10 @@ and `location`. The fields actually added depend on what has been found and whic
`country_iso_code`, `country_name` and `continent_name`. The fields actually added depend on what has been found and which properties
were configured in `properties`.
* If the GeoLite2 ASN database is used, then the following fields may be added under the `target_field`: `ip`,
`asn`, and `organization_name`. The fields actually added depend on what has been found and which properties were configured
`asn`, `organization_name` and `network`. The fields actually added depend on what has been found and which properties were configured
in `properties`.
Here is an example that uses the default city database and adds the geographical information to the `geoip` field based on the `ip` field:
[source,console]

View File

@ -19,6 +19,7 @@
package org.elasticsearch.ingest.geoip;
import com.maxmind.db.Network;
import com.maxmind.geoip2.exception.AddressNotFoundException;
import com.maxmind.geoip2.model.AsnResponse;
import com.maxmind.geoip2.model.CityResponse;
@ -345,6 +346,7 @@ public final class GeoIpProcessor extends AbstractProcessor {
Integer asn = response.getAutonomousSystemNumber();
String organization_name = response.getAutonomousSystemOrganization();
Network network = response.getNetwork();
Map<String, Object> geoData = new HashMap<>();
for (Property property : this.properties) {
@ -362,6 +364,11 @@ public final class GeoIpProcessor extends AbstractProcessor {
geoData.put("organization_name", organization_name);
}
break;
case NETWORK:
if (network != null) {
geoData.put("network", network.toString());
}
break;
}
}
return geoData;
@ -376,7 +383,7 @@ public final class GeoIpProcessor extends AbstractProcessor {
Property.CONTINENT_NAME, Property.COUNTRY_NAME, Property.COUNTRY_ISO_CODE
));
static final Set<Property> DEFAULT_ASN_PROPERTIES = Collections.unmodifiableSet(EnumSet.of(
Property.IP, Property.ASN, Property.ORGANIZATION_NAME
Property.IP, Property.ASN, Property.ORGANIZATION_NAME, Property.NETWORK
));
private final Map<String, DatabaseReaderLazyLoader> databaseReaders;
@ -464,7 +471,8 @@ public final class GeoIpProcessor extends AbstractProcessor {
TIMEZONE,
LOCATION,
ASN,
ORGANIZATION_NAME;
ORGANIZATION_NAME,
NETWORK;
static final EnumSet<Property> ALL_CITY_PROPERTIES = EnumSet.of(
Property.IP, Property.COUNTRY_ISO_CODE, Property.COUNTRY_NAME, Property.CONTINENT_NAME,
@ -475,7 +483,7 @@ public final class GeoIpProcessor extends AbstractProcessor {
Property.IP, Property.CONTINENT_NAME, Property.COUNTRY_NAME, Property.COUNTRY_ISO_CODE
);
static final EnumSet<Property> ALL_ASN_PROPERTIES = EnumSet.of(
Property.IP, Property.ASN, Property.ORGANIZATION_NAME
Property.IP, Property.ASN, Property.ORGANIZATION_NAME, Property.NETWORK
);
public static Property parseProperty(String databaseType, String value) {

View File

@ -189,7 +189,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
config.put("properties", Collections.singletonList(cityProperty));
Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config));
assertThat(e.getMessage(), equalTo("[properties] illegal property value [" + cityProperty +
"]. valid values are [IP, ASN, ORGANIZATION_NAME]"));
"]. valid values are [IP, ASN, ORGANIZATION_NAME, NETWORK]"));
}
public void testBuildNonExistingDbFile() throws Exception {

View File

@ -209,10 +209,11 @@ public class GeoIpProcessorTests extends ESTestCase {
assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo(ip));
@SuppressWarnings("unchecked")
Map<String, Object> geoData = (Map<String, Object>) ingestDocument.getSourceAndMetadata().get("target_field");
assertThat(geoData.size(), equalTo(3));
assertThat(geoData.size(), equalTo(4));
assertThat(geoData.get("ip"), equalTo(ip));
assertThat(geoData.get("asn"), equalTo(1136));
assertThat(geoData.get("organization_name"), equalTo("KPN B.V."));
assertThat(geoData.get("network"), equalTo("82.168.0.0/14"));
}
public void testAddressIsNotInTheDatabase() throws Exception {

View File

@ -304,7 +304,8 @@
index: test
id: 1
- match: { _source.field1: "82.171.64.0" }
- length: { _source.geoip: 3 }
- length: { _source.geoip: 4 }
- match: { _source.geoip.ip: "82.171.64.0" }
- match: { _source.geoip.asn: 1136 }
- match: { _source.geoip.organization_name: "KPN B.V." }
- match: { _source.geoip.network: "82.168.0.0/14" }