This adds the network property from the MaxMind Geo ASN database. This enables analysis of IP data based on the subnets that MaxMind have previously identified for ASN networks. closes #60942 Co-authored-by: Peter Ansell <p_ansell@yahoo.com>
This commit is contained in:
parent
95242eccee
commit
0795f4b898
|
@ -40,9 +40,10 @@ and `location`. The fields actually added depend on what has been found and whic
|
||||||
`country_iso_code`, `country_name` and `continent_name`. The fields actually added depend on what has been found and which properties
|
`country_iso_code`, `country_name` and `continent_name`. The fields actually added depend on what has been found and which properties
|
||||||
were configured in `properties`.
|
were configured in `properties`.
|
||||||
* If the GeoLite2 ASN database is used, then the following fields may be added under the `target_field`: `ip`,
|
* If the GeoLite2 ASN database is used, then the following fields may be added under the `target_field`: `ip`,
|
||||||
`asn`, and `organization_name`. The fields actually added depend on what has been found and which properties were configured
|
`asn`, `organization_name` and `network`. The fields actually added depend on what has been found and which properties were configured
|
||||||
in `properties`.
|
in `properties`.
|
||||||
|
|
||||||
|
|
||||||
Here is an example that uses the default city database and adds the geographical information to the `geoip` field based on the `ip` field:
|
Here is an example that uses the default city database and adds the geographical information to the `geoip` field based on the `ip` field:
|
||||||
|
|
||||||
[source,console]
|
[source,console]
|
||||||
|
|
|
@ -19,6 +19,7 @@
|
||||||
|
|
||||||
package org.elasticsearch.ingest.geoip;
|
package org.elasticsearch.ingest.geoip;
|
||||||
|
|
||||||
|
import com.maxmind.db.Network;
|
||||||
import com.maxmind.geoip2.exception.AddressNotFoundException;
|
import com.maxmind.geoip2.exception.AddressNotFoundException;
|
||||||
import com.maxmind.geoip2.model.AsnResponse;
|
import com.maxmind.geoip2.model.AsnResponse;
|
||||||
import com.maxmind.geoip2.model.CityResponse;
|
import com.maxmind.geoip2.model.CityResponse;
|
||||||
|
@ -345,6 +346,7 @@ public final class GeoIpProcessor extends AbstractProcessor {
|
||||||
|
|
||||||
Integer asn = response.getAutonomousSystemNumber();
|
Integer asn = response.getAutonomousSystemNumber();
|
||||||
String organization_name = response.getAutonomousSystemOrganization();
|
String organization_name = response.getAutonomousSystemOrganization();
|
||||||
|
Network network = response.getNetwork();
|
||||||
|
|
||||||
Map<String, Object> geoData = new HashMap<>();
|
Map<String, Object> geoData = new HashMap<>();
|
||||||
for (Property property : this.properties) {
|
for (Property property : this.properties) {
|
||||||
|
@ -362,6 +364,11 @@ public final class GeoIpProcessor extends AbstractProcessor {
|
||||||
geoData.put("organization_name", organization_name);
|
geoData.put("organization_name", organization_name);
|
||||||
}
|
}
|
||||||
break;
|
break;
|
||||||
|
case NETWORK:
|
||||||
|
if (network != null) {
|
||||||
|
geoData.put("network", network.toString());
|
||||||
|
}
|
||||||
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return geoData;
|
return geoData;
|
||||||
|
@ -376,7 +383,7 @@ public final class GeoIpProcessor extends AbstractProcessor {
|
||||||
Property.CONTINENT_NAME, Property.COUNTRY_NAME, Property.COUNTRY_ISO_CODE
|
Property.CONTINENT_NAME, Property.COUNTRY_NAME, Property.COUNTRY_ISO_CODE
|
||||||
));
|
));
|
||||||
static final Set<Property> DEFAULT_ASN_PROPERTIES = Collections.unmodifiableSet(EnumSet.of(
|
static final Set<Property> DEFAULT_ASN_PROPERTIES = Collections.unmodifiableSet(EnumSet.of(
|
||||||
Property.IP, Property.ASN, Property.ORGANIZATION_NAME
|
Property.IP, Property.ASN, Property.ORGANIZATION_NAME, Property.NETWORK
|
||||||
));
|
));
|
||||||
|
|
||||||
private final Map<String, DatabaseReaderLazyLoader> databaseReaders;
|
private final Map<String, DatabaseReaderLazyLoader> databaseReaders;
|
||||||
|
@ -464,7 +471,8 @@ public final class GeoIpProcessor extends AbstractProcessor {
|
||||||
TIMEZONE,
|
TIMEZONE,
|
||||||
LOCATION,
|
LOCATION,
|
||||||
ASN,
|
ASN,
|
||||||
ORGANIZATION_NAME;
|
ORGANIZATION_NAME,
|
||||||
|
NETWORK;
|
||||||
|
|
||||||
static final EnumSet<Property> ALL_CITY_PROPERTIES = EnumSet.of(
|
static final EnumSet<Property> ALL_CITY_PROPERTIES = EnumSet.of(
|
||||||
Property.IP, Property.COUNTRY_ISO_CODE, Property.COUNTRY_NAME, Property.CONTINENT_NAME,
|
Property.IP, Property.COUNTRY_ISO_CODE, Property.COUNTRY_NAME, Property.CONTINENT_NAME,
|
||||||
|
@ -475,7 +483,7 @@ public final class GeoIpProcessor extends AbstractProcessor {
|
||||||
Property.IP, Property.CONTINENT_NAME, Property.COUNTRY_NAME, Property.COUNTRY_ISO_CODE
|
Property.IP, Property.CONTINENT_NAME, Property.COUNTRY_NAME, Property.COUNTRY_ISO_CODE
|
||||||
);
|
);
|
||||||
static final EnumSet<Property> ALL_ASN_PROPERTIES = EnumSet.of(
|
static final EnumSet<Property> ALL_ASN_PROPERTIES = EnumSet.of(
|
||||||
Property.IP, Property.ASN, Property.ORGANIZATION_NAME
|
Property.IP, Property.ASN, Property.ORGANIZATION_NAME, Property.NETWORK
|
||||||
);
|
);
|
||||||
|
|
||||||
public static Property parseProperty(String databaseType, String value) {
|
public static Property parseProperty(String databaseType, String value) {
|
||||||
|
|
|
@ -189,7 +189,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
|
||||||
config.put("properties", Collections.singletonList(cityProperty));
|
config.put("properties", Collections.singletonList(cityProperty));
|
||||||
Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config));
|
Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config));
|
||||||
assertThat(e.getMessage(), equalTo("[properties] illegal property value [" + cityProperty +
|
assertThat(e.getMessage(), equalTo("[properties] illegal property value [" + cityProperty +
|
||||||
"]. valid values are [IP, ASN, ORGANIZATION_NAME]"));
|
"]. valid values are [IP, ASN, ORGANIZATION_NAME, NETWORK]"));
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testBuildNonExistingDbFile() throws Exception {
|
public void testBuildNonExistingDbFile() throws Exception {
|
||||||
|
|
|
@ -209,10 +209,11 @@ public class GeoIpProcessorTests extends ESTestCase {
|
||||||
assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo(ip));
|
assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo(ip));
|
||||||
@SuppressWarnings("unchecked")
|
@SuppressWarnings("unchecked")
|
||||||
Map<String, Object> geoData = (Map<String, Object>) ingestDocument.getSourceAndMetadata().get("target_field");
|
Map<String, Object> geoData = (Map<String, Object>) ingestDocument.getSourceAndMetadata().get("target_field");
|
||||||
assertThat(geoData.size(), equalTo(3));
|
assertThat(geoData.size(), equalTo(4));
|
||||||
assertThat(geoData.get("ip"), equalTo(ip));
|
assertThat(geoData.get("ip"), equalTo(ip));
|
||||||
assertThat(geoData.get("asn"), equalTo(1136));
|
assertThat(geoData.get("asn"), equalTo(1136));
|
||||||
assertThat(geoData.get("organization_name"), equalTo("KPN B.V."));
|
assertThat(geoData.get("organization_name"), equalTo("KPN B.V."));
|
||||||
|
assertThat(geoData.get("network"), equalTo("82.168.0.0/14"));
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testAddressIsNotInTheDatabase() throws Exception {
|
public void testAddressIsNotInTheDatabase() throws Exception {
|
||||||
|
|
|
@ -304,7 +304,8 @@
|
||||||
index: test
|
index: test
|
||||||
id: 1
|
id: 1
|
||||||
- match: { _source.field1: "82.171.64.0" }
|
- match: { _source.field1: "82.171.64.0" }
|
||||||
- length: { _source.geoip: 3 }
|
- length: { _source.geoip: 4 }
|
||||||
- match: { _source.geoip.ip: "82.171.64.0" }
|
- match: { _source.geoip.ip: "82.171.64.0" }
|
||||||
- match: { _source.geoip.asn: 1136 }
|
- match: { _source.geoip.asn: 1136 }
|
||||||
- match: { _source.geoip.organization_name: "KPN B.V." }
|
- match: { _source.geoip.organization_name: "KPN B.V." }
|
||||||
|
- match: { _source.geoip.network: "82.168.0.0/14" }
|
||||||
|
|
Loading…
Reference in New Issue