ingest: Upgraded the geolite2 databases.

docs/plugins/ingest-geoip.asciidoc

@@ -6,7 +6,7 @@ This processor adds this information by default under the `geoip` field. The `ge
 IPv6 addresses.
 
 The ingest-geoip plugin ships by default with the GeoLite2 City and GeoLite2 Country geoip2 databases from Maxmind made available
-under the CCA-ShareAlike 3.0 license. For more details see, http://dev.maxmind.com/geoip/geoip2/geolite2/
+under the CCA-ShareAlike 4.0 license. For more details see, http://dev.maxmind.com/geoip/geoip2/geolite2/
 
 The GeoIP processor can run with other geoip2 databases from Maxmind. The files must be copied into the geoip config directory,
 and the `database_file` option should be used to specify the filename of the custom database. Custom database files must be compressed
@@ -76,9 +76,7 @@ Which returns:
     "geoip": {
       "continent_name": "North America",
       "country_iso_code": "US",
-      "region_name": "California",
+      "location": { "lat": 37.751, "lon": -97.822 }
-      "city_name": "Mountain View",
-      "location": { "lat": 37.386, "lon": -122.0838 }
     }
   }
 }
@@ -137,7 +135,7 @@ returns this:
 Not all IP addresses find geo information from the database, When this
 occurs, no `target_field` is inserted into the document.
 
-Here is an example of what documents will be indexed as when information for "93.114.45.13"
+Here is an example of what documents will be indexed as when information for "80.231.5.0"
 cannot be found:
 
 [source,js]
@@ -155,7 +153,7 @@ PUT _ingest/pipeline/geoip
 }
 PUT my_index/my_type/my_id?pipeline=geoip
 {
-  "ip": "93.114.45.13"
+  "ip": "80.231.5.0"
 }
 GET my_index/my_type/my_id
 --------------------------------------------------
@@ -172,7 +170,7 @@ Which returns:
   "_id": "my_id",
   "_version": 1,
   "_source": {
-    "ip": "93.114.45.13"
+    "ip": "80.231.5.0"
   }
 }
 --------------------------------------------------

plugins/ingest-geoip/build.gradle

@@ -29,13 +29,16 @@ dependencies {
   compile('com.fasterxml.jackson.core:jackson-databind:2.8.2')
   compile('com.maxmind.db:maxmind-db:1.2.1')
 
-  testCompile 'org.elasticsearch:geolite2-databases:20160608'
+  testCompile 'org.elasticsearch:geolite2-databases:20171206'
 }
 
 task copyDefaultGeoIp2DatabaseFiles(type: Copy) {
   from { zipTree(configurations.testCompile.files.find { it.name.contains('geolite2-databases')}) }
   into "${project.buildDir}/ingest-geoip"
-  include "*.mmdb.gz"
+
+  // For now, do not include GeoLite2-ASN.mmdb.gz file, because it isn't used yet:
+  include "GeoLite2-City.mmdb.gz"
+  include "GeoLite2-Country.mmdb.gz"
 }
 
 project.bundlePlugin.dependsOn(copyDefaultGeoIp2DatabaseFiles)

plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java

@@ -52,17 +52,14 @@ public class GeoIpProcessorTests extends ESTestCase {
         assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo("8.8.8.8"));
         @SuppressWarnings("unchecked")
         Map<String, Object> geoData = (Map<String, Object>) ingestDocument.getSourceAndMetadata().get("target_field");
-        assertThat(geoData.size(), equalTo(8));
+        assertThat(geoData.size(), equalTo(5));
         assertThat(geoData.get("ip"), equalTo("8.8.8.8"));
         assertThat(geoData.get("country_iso_code"), equalTo("US"));
         assertThat(geoData.get("country_name"), equalTo("United States"));
         assertThat(geoData.get("continent_name"), equalTo("North America"));
-        assertThat(geoData.get("region_name"), equalTo("California"));
-        assertThat(geoData.get("city_name"), equalTo("Mountain View"));
-        assertThat(geoData.get("timezone"), equalTo("America/Los_Angeles"));
         Map<String, Object> location = new HashMap<>();
-        location.put("lat", 37.386d);
+        location.put("lat", 37.751d);
-        location.put("lon", -122.0838d);
+        location.put("lon", -97.822d);
         assertThat(geoData.get("location"), equalTo(location));
     }
 
@@ -142,15 +139,15 @@ public class GeoIpProcessorTests extends ESTestCase {
             new DatabaseReader.Builder(database).build(), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class), false);
 
         Map<String, Object> document = new HashMap<>();
-        document.put("source_field", "93.114.45.13");
+        document.put("source_field", "80.231.5.0");
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
         processor.execute(ingestDocument);
 
-        assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo("93.114.45.13"));
+        assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo("80.231.5.0"));
         @SuppressWarnings("unchecked")
         Map<String, Object> geoData = (Map<String, Object>) ingestDocument.getSourceAndMetadata().get("target_field");
         assertThat(geoData.size(), equalTo(1));
-        assertThat(geoData.get("ip"), equalTo("93.114.45.13"));
+        assertThat(geoData.get("ip"), equalTo("80.231.5.0"));
     }
 
     public void testCountry() throws Exception {
@@ -179,15 +176,15 @@ public class GeoIpProcessorTests extends ESTestCase {
             new DatabaseReader.Builder(database).build(), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class), false);
 
         Map<String, Object> document = new HashMap<>();
-        document.put("source_field", "93.114.45.13");
+        document.put("source_field", "80.231.5.0");
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
         processor.execute(ingestDocument);
 
-        assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo("93.114.45.13"));
+        assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo("80.231.5.0"));
         @SuppressWarnings("unchecked")
         Map<String, Object> geoData = (Map<String, Object>) ingestDocument.getSourceAndMetadata().get("target_field");
         assertThat(geoData.size(), equalTo(1));
-        assertThat(geoData.get("ip"), equalTo("93.114.45.13"));
+        assertThat(geoData.get("ip"), equalTo("80.231.5.0"));
     }
 
     public void testAddressIsNotInTheDatabase() throws Exception {

plugins/ingest-geoip/src/test/resources/rest-api-spec/test/ingest_geoip/20_geoip_processor.yml

@@ -164,14 +164,14 @@
         type: test
         id: 1
         pipeline: "my_pipeline"
-        body: { field1: "93.114.45.13" }
+        body: { field1: "80.231.5.0" }
 
   - do:
       get:
         index: test
         type: test
         id: 1
-  - match: { _source.field1: "93.114.45.13" }
+  - match: { _source.field1: "80.231.5.0" }
   - is_false: _source.geoip
 
   - do: