ingest: Upgraded the geolite2 databases.
This commit is contained in:
Martijn van Groningen
parent
0f80e7c5f6
commit
4585cc8312
|
|
@ -6,7 +6,7 @@ This processor adds this information by default under the `geoip` field. The `ge
|
|||
IPv6 addresses.
|
||||
|
||||
The ingest-geoip plugin ships by default with the GeoLite2 City and GeoLite2 Country geoip2 databases from Maxmind made available
|
||||
under the CCA-ShareAlike 3.0 license. For more details see, http://dev.maxmind.com/geoip/geoip2/geolite2/
|
||||
under the CCA-ShareAlike 4.0 license. For more details see, http://dev.maxmind.com/geoip/geoip2/geolite2/
|
||||
|
||||
The GeoIP processor can run with other geoip2 databases from Maxmind. The files must be copied into the geoip config directory,
|
||||
and the `database_file` option should be used to specify the filename of the custom database. Custom database files must be compressed
|
||||
|
|
@ -76,9 +76,7 @@ Which returns:
|
|||
"geoip": {
|
||||
"continent_name": "North America",
|
||||
"country_iso_code": "US",
|
||||
"region_name": "California",
|
||||
"city_name": "Mountain View",
|
||||
"location": { "lat": 37.386, "lon": -122.0838 }
|
||||
"location": { "lat": 37.751, "lon": -97.822 }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -137,7 +135,7 @@ returns this:
|
|||
Not all IP addresses find geo information from the database, When this
|
||||
occurs, no `target_field` is inserted into the document.
|
||||
|
||||
Here is an example of what documents will be indexed as when information for "93.114.45.13"
|
||||
Here is an example of what documents will be indexed as when information for "80.231.5.0"
|
||||
cannot be found:
|
||||
|
||||
[source,js]
|
||||
|
|
@ -155,7 +153,7 @@ PUT _ingest/pipeline/geoip
|
|||
}
|
||||
PUT my_index/my_type/my_id?pipeline=geoip
|
||||
{
|
||||
"ip": "93.114.45.13"
|
||||
"ip": "80.231.5.0"
|
||||
}
|
||||
GET my_index/my_type/my_id
|
||||
--------------------------------------------------
|
||||
|
|
@ -172,7 +170,7 @@ Which returns:
|
|||
"_id": "my_id",
|
||||
"_version": 1,
|
||||
"_source": {
|
||||
"ip": "93.114.45.13"
|
||||
"ip": "80.231.5.0"
|
||||
}
|
||||
}
|
||||
--------------------------------------------------
|
||||
|
|
|
|||
|
|
@ -29,13 +29,16 @@ dependencies {
|
|||
compile('com.fasterxml.jackson.core:jackson-databind:2.8.2')
|
||||
compile('com.maxmind.db:maxmind-db:1.2.1')
|
||||
|
||||
testCompile 'org.elasticsearch:geolite2-databases:20160608'
|
||||
testCompile 'org.elasticsearch:geolite2-databases:20171206'
|
||||
}
|
||||
|
||||
task copyDefaultGeoIp2DatabaseFiles(type: Copy) {
|
||||
from { zipTree(configurations.testCompile.files.find { it.name.contains('geolite2-databases')}) }
|
||||
into "${project.buildDir}/ingest-geoip"
|
||||
include "*.mmdb.gz"
|
||||
|
||||
// For now, do not include GeoLite2-ASN.mmdb.gz file, because it isn't used yet:
|
||||
include "GeoLite2-City.mmdb.gz"
|
||||
include "GeoLite2-Country.mmdb.gz"
|
||||
}
|
||||
|
||||
project.bundlePlugin.dependsOn(copyDefaultGeoIp2DatabaseFiles)
|
||||
|
|
|
|||
|
|
@ -52,17 +52,14 @@ public class GeoIpProcessorTests extends ESTestCase {
|
|||
assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo("8.8.8.8"));
|
||||
@SuppressWarnings("unchecked")
|
||||
Map<String, Object> geoData = (Map<String, Object>) ingestDocument.getSourceAndMetadata().get("target_field");
|
||||
assertThat(geoData.size(), equalTo(8));
|
||||
assertThat(geoData.size(), equalTo(5));
|
||||
assertThat(geoData.get("ip"), equalTo("8.8.8.8"));
|
||||
assertThat(geoData.get("country_iso_code"), equalTo("US"));
|
||||
assertThat(geoData.get("country_name"), equalTo("United States"));
|
||||
assertThat(geoData.get("continent_name"), equalTo("North America"));
|
||||
assertThat(geoData.get("region_name"), equalTo("California"));
|
||||
assertThat(geoData.get("city_name"), equalTo("Mountain View"));
|
||||
assertThat(geoData.get("timezone"), equalTo("America/Los_Angeles"));
|
||||
Map<String, Object> location = new HashMap<>();
|
||||
location.put("lat", 37.386d);
|
||||
location.put("lon", -122.0838d);
|
||||
location.put("lat", 37.751d);
|
||||
location.put("lon", -97.822d);
|
||||
assertThat(geoData.get("location"), equalTo(location));
|
||||
}
|
||||
|
||||
|
|
@ -142,15 +139,15 @@ public class GeoIpProcessorTests extends ESTestCase {
|
|||
new DatabaseReader.Builder(database).build(), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class), false);
|
||||
|
||||
Map<String, Object> document = new HashMap<>();
|
||||
document.put("source_field", "93.114.45.13");
|
||||
document.put("source_field", "80.231.5.0");
|
||||
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
|
||||
processor.execute(ingestDocument);
|
||||
|
||||
assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo("93.114.45.13"));
|
||||
assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo("80.231.5.0"));
|
||||
@SuppressWarnings("unchecked")
|
||||
Map<String, Object> geoData = (Map<String, Object>) ingestDocument.getSourceAndMetadata().get("target_field");
|
||||
assertThat(geoData.size(), equalTo(1));
|
||||
assertThat(geoData.get("ip"), equalTo("93.114.45.13"));
|
||||
assertThat(geoData.get("ip"), equalTo("80.231.5.0"));
|
||||
}
|
||||
|
||||
public void testCountry() throws Exception {
|
||||
|
|
@ -179,15 +176,15 @@ public class GeoIpProcessorTests extends ESTestCase {
|
|||
new DatabaseReader.Builder(database).build(), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class), false);
|
||||
|
||||
Map<String, Object> document = new HashMap<>();
|
||||
document.put("source_field", "93.114.45.13");
|
||||
document.put("source_field", "80.231.5.0");
|
||||
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
|
||||
processor.execute(ingestDocument);
|
||||
|
||||
assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo("93.114.45.13"));
|
||||
assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo("80.231.5.0"));
|
||||
@SuppressWarnings("unchecked")
|
||||
Map<String, Object> geoData = (Map<String, Object>) ingestDocument.getSourceAndMetadata().get("target_field");
|
||||
assertThat(geoData.size(), equalTo(1));
|
||||
assertThat(geoData.get("ip"), equalTo("93.114.45.13"));
|
||||
assertThat(geoData.get("ip"), equalTo("80.231.5.0"));
|
||||
}
|
||||
|
||||
public void testAddressIsNotInTheDatabase() throws Exception {
|
||||
|
|
|
|||
|
|
@ -164,14 +164,14 @@
|
|||
type: test
|
||||
id: 1
|
||||
pipeline: "my_pipeline"
|
||||
body: { field1: "93.114.45.13" }
|
||||
body: { field1: "80.231.5.0" }
|
||||
|
||||
- do:
|
||||
get:
|
||||
index: test
|
||||
type: test
|
||||
id: 1
|
||||
- match: { _source.field1: "93.114.45.13" }
|
||||
- match: { _source.field1: "80.231.5.0" }
|
||||
- is_false: _source.geoip
|
||||
|
||||
- do:
|
||||
|
|
|
|||
Loading…
Reference in New Issue