honeymoose/OpenSearch
1
0
Fork 0
mirror of https://github.com/honeymoose/OpenSearch.git synced 2025-02-20 03:45:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

Reduce heap-memory usage of ingest-geoip plugin (#28963)

Browse Source 
With this commit we reduce heap usage of the ingest-geoip plugin by
memory-mapping the database files. Previously, we have stored these
files gzip-compressed but this has resulted that data are loaded on the
heap.

Closes #28782
This commit is contained in:
Daniel Mitterdorfer 2018-03-12 08:07:33 +01:00 committed by GitHub
parent aeaebddf4a
commit 0d78a5890e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 66 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
docs/plugins/ingest-geoip.asciidoc
View File

@ -9,8 +9,8 @@ The ingest-geoip plugin ships by default with the GeoLite2 City, GeoLite2 Countr
under the CCA-ShareAlike 4.0 license. For more details see, http://dev.maxmind.com/geoip/geoip2/geolite2/ under the CCA-ShareAlike 4.0 license. For more details see, http://dev.maxmind.com/geoip/geoip2/geolite2/
The GeoIP processor can run with other geoip2 databases from Maxmind. The files must be copied into the geoip config directory, The GeoIP processor can run with other geoip2 databases from Maxmind. The files must be copied into the geoip config directory,
and the `database_file` option should be used to specify the filename of the custom database. Custom database files must be compressed and the `database_file` option should be used to specify the filename of the custom database. Custom database files must be stored
with gzip. The geoip config directory is located at `$ES_HOME/config/ingest-geoip` and holds the shipped databases too. uncompressed. The geoip config directory is located at `$ES_HOME/config/ingest-geoip` and holds the shipped databases too.
:plugin_name: ingest-geoip :plugin_name: ingest-geoip
include::install_remove.asciidoc[] include::install_remove.asciidoc[]
@ -25,7 +25,7 @@ include::install_remove.asciidoc[]
| Name | Required | Default | Description | Name | Required | Default | Description
| `field` | yes | - | The field to get the ip address from for the geographical lookup. | `field` | yes | - | The field to get the ip address from for the geographical lookup.
| `target_field` | no | geoip | The field that will hold the geographical information looked up from the Maxmind database. | `target_field` | no | geoip | The field that will hold the geographical information looked up from the Maxmind database.
| `database_file` | no | GeoLite2-City.mmdb | The database filename in the geoip config directory. The ingest-geoip plugin ships with the GeoLite2-City.mmdb.gz and GeoLite2-Country.mmdb.gz files. | `database_file` | no | GeoLite2-City.mmdb | The database filename in the geoip config directory. The ingest-geoip plugin ships with the GeoLite2-City.mmdb, GeoLite2-Country.mmdb and GeoLite2-ASN.mmdb files.
| `properties` | no | [`continent_name`, `country_iso_code`, `region_name`, `city_name`, `location`] * | Controls what properties are added to the `target_field` based on the geoip lookup. | `properties` | no | [`continent_name`, `country_iso_code`, `region_name`, `city_name`, `location`] * | Controls what properties are added to the `target_field` based on the geoip lookup.
| `ignore_missing` | no | `false` | If `true` and `field` does not exist, the processor quietly exits without modifying the document | `ignore_missing` | no | `false` | If `true` and `field` does not exist, the processor quietly exits without modifying the document
|====== |======
@ -101,7 +101,7 @@ PUT _ingest/pipeline/geoip
"geoip" : { "geoip" : {
"field" : "ip", "field" : "ip",
"target_field" : "geo", "target_field" : "geo",
"database_file" : "GeoLite2-Country.mmdb.gz" "database_file" : "GeoLite2-Country.mmdb"
} }
} }
] ]

4
plugins/ingest-geoip/build.gradle
View File

@ -30,13 +30,13 @@ dependencies {
compile("com.fasterxml.jackson.core:jackson-databind:${versions.jackson}") compile("com.fasterxml.jackson.core:jackson-databind:${versions.jackson}")
compile('com.maxmind.db:maxmind-db:1.2.2') compile('com.maxmind.db:maxmind-db:1.2.2')
testCompile 'org.elasticsearch:geolite2-databases:20171206' testCompile 'org.elasticsearch:geolite2-databases:20180303'
} }
task copyDefaultGeoIp2DatabaseFiles(type: Copy) { task copyDefaultGeoIp2DatabaseFiles(type: Copy) {
from { zipTree(configurations.testCompile.files.find { it.name.contains('geolite2-databases')}) } from { zipTree(configurations.testCompile.files.find { it.name.contains('geolite2-databases')}) }
into "${project.buildDir}/ingest-geoip" into "${project.buildDir}/ingest-geoip"
include "*.mmdb.gz" include "*.mmdb"
} }
project.bundlePlugin.dependsOn(copyDefaultGeoIp2DatabaseFiles) project.bundlePlugin.dependsOn(copyDefaultGeoIp2DatabaseFiles)

6
plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java
View File

@ -37,7 +37,6 @@ import org.elasticsearch.ingest.AbstractProcessor;
import org.elasticsearch.ingest.IngestDocument; import org.elasticsearch.ingest.IngestDocument;
import org.elasticsearch.ingest.Processor; import org.elasticsearch.ingest.Processor;
import java.io.IOException;
import java.net.InetAddress; import java.net.InetAddress;
import java.security.AccessController; import java.security.AccessController;
import java.security.PrivilegedAction; import java.security.PrivilegedAction;
@ -68,8 +67,7 @@ public final class GeoIpProcessor extends AbstractProcessor {
private final Set<Property> properties; private final Set<Property> properties;
private final boolean ignoreMissing; private final boolean ignoreMissing;
GeoIpProcessor(String tag, String field, DatabaseReader dbReader, String targetField, Set<Property> properties, GeoIpProcessor(String tag, String field, DatabaseReader dbReader, String targetField, Set<Property> properties, boolean ignoreMissing) {
boolean ignoreMissing) throws IOException {
super(tag); super(tag);
this.field = field; this.field = field;
this.targetField = targetField; this.targetField = targetField;
@ -323,7 +321,7 @@ public final class GeoIpProcessor extends AbstractProcessor {
Map<String, Object> config) throws Exception { Map<String, Object> config) throws Exception {
String ipField = readStringProperty(TYPE, processorTag, config, "field"); String ipField = readStringProperty(TYPE, processorTag, config, "field");
String targetField = readStringProperty(TYPE, processorTag, config, "target_field", "geoip"); String targetField = readStringProperty(TYPE, processorTag, config, "target_field", "geoip");
String databaseFile = readStringProperty(TYPE, processorTag, config, "database_file", "GeoLite2-City.mmdb.gz"); String databaseFile = readStringProperty(TYPE, processorTag, config, "database_file", "GeoLite2-City.mmdb");
List<String> propertyNames = readOptionalList(TYPE, processorTag, config, "properties"); List<String> propertyNames = readOptionalList(TYPE, processorTag, config, "properties");
boolean ignoreMissing = readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false); boolean ignoreMissing = readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false);

30
plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java
View File

@ -21,8 +21,11 @@ package org.elasticsearch.ingest.geoip;
import com.maxmind.db.NoCache; import com.maxmind.db.NoCache;
import com.maxmind.db.NodeCache; import com.maxmind.db.NodeCache;
import com.maxmind.db.Reader;
import com.maxmind.geoip2.DatabaseReader; import com.maxmind.geoip2.DatabaseReader;
import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.IOUtils;
import org.elasticsearch.common.Booleans;
import org.elasticsearch.common.SuppressForbidden;
import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.ingest.Processor; import org.elasticsearch.ingest.Processor;
import org.elasticsearch.plugins.IngestPlugin; import org.elasticsearch.plugins.IngestPlugin;
@ -30,11 +33,9 @@ import org.elasticsearch.plugins.Plugin;
import java.io.Closeable; import java.io.Closeable;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
import java.nio.file.PathMatcher; import java.nio.file.PathMatcher;
import java.nio.file.StandardOpenOption;
import java.util.Arrays; import java.util.Arrays;
import java.util.Collections; import java.util.Collections;
import java.util.HashMap; import java.util.HashMap;
@ -42,7 +43,6 @@ import java.util.Iterator;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.stream.Stream; import java.util.stream.Stream;
import java.util.zip.GZIPInputStream;
public class IngestGeoIpPlugin extends Plugin implements IngestPlugin, Closeable { public class IngestGeoIpPlugin extends Plugin implements IngestPlugin, Closeable {
public static final Setting<Long> CACHE_SIZE = public static final Setting<Long> CACHE_SIZE =
@ -80,21 +80,26 @@ public class IngestGeoIpPlugin extends Plugin implements IngestPlugin, Closeable
if (Files.exists(geoIpConfigDirectory) == false && Files.isDirectory(geoIpConfigDirectory)) { if (Files.exists(geoIpConfigDirectory) == false && Files.isDirectory(geoIpConfigDirectory)) {
throw new IllegalStateException("the geoip directory [" + geoIpConfigDirectory + "] containing databases doesn't exist"); throw new IllegalStateException("the geoip directory [" + geoIpConfigDirectory + "] containing databases doesn't exist");
} }
boolean loadDatabaseOnHeap = Booleans.parseBoolean(System.getProperty("es.geoip.load_db_on_heap", "false"));
Map<String, DatabaseReaderLazyLoader> databaseReaders = new HashMap<>(); Map<String, DatabaseReaderLazyLoader> databaseReaders = new HashMap<>();
try (Stream<Path> databaseFiles = Files.list(geoIpConfigDirectory)) { try (Stream<Path> databaseFiles = Files.list(geoIpConfigDirectory)) {
PathMatcher pathMatcher = geoIpConfigDirectory.getFileSystem().getPathMatcher("glob:**.mmdb.gz"); PathMatcher pathMatcher = geoIpConfigDirectory.getFileSystem().getPathMatcher("glob:**.mmdb");
// Use iterator instead of forEach otherwise IOException needs to be caught twice... // Use iterator instead of forEach otherwise IOException needs to be caught twice...
Iterator<Path> iterator = databaseFiles.iterator(); Iterator<Path> iterator = databaseFiles.iterator();
while (iterator.hasNext()) { while (iterator.hasNext()) {
Path databasePath = iterator.next(); Path databasePath = iterator.next();
if (Files.isRegularFile(databasePath) && pathMatcher.matches(databasePath)) { if (Files.isRegularFile(databasePath) && pathMatcher.matches(databasePath)) {
String databaseFileName = databasePath.getFileName().toString(); String databaseFileName = databasePath.getFileName().toString();
DatabaseReaderLazyLoader holder = new DatabaseReaderLazyLoader(databaseFileName, () -> { DatabaseReaderLazyLoader holder = new DatabaseReaderLazyLoader(databaseFileName,
try (InputStream inputStream = new GZIPInputStream(Files.newInputStream(databasePath, StandardOpenOption.READ))) { () -> {
return new DatabaseReader.Builder(inputStream).withCache(cache).build(); DatabaseReader.Builder builder = createDatabaseBuilder(databasePath).withCache(cache);
} if (loadDatabaseOnHeap) {
}); builder.fileMode(Reader.FileMode.MEMORY);
} else {
builder.fileMode(Reader.FileMode.MEMORY_MAPPED);
}
return builder.build();
});
databaseReaders.put(databaseFileName, holder); databaseReaders.put(databaseFileName, holder);
} }
} }
@ -102,6 +107,11 @@ public class IngestGeoIpPlugin extends Plugin implements IngestPlugin, Closeable
return Collections.unmodifiableMap(databaseReaders); return Collections.unmodifiableMap(databaseReaders);
} }
@SuppressForbidden(reason = "Maxmind API requires java.io.File")
private static DatabaseReader.Builder createDatabaseBuilder(Path databasePath) {
return new DatabaseReader.Builder(databasePath.toFile());
}
@Override @Override
public void close() throws IOException { public void close() throws IOException {
if (databaseReaders != null) { if (databaseReaders != null) {

44
plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java
View File

@ -54,12 +54,12 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
Path configDir = createTempDir(); Path configDir = createTempDir();
Path geoIpConfigDir = configDir.resolve("ingest-geoip"); Path geoIpConfigDir = configDir.resolve("ingest-geoip");
Files.createDirectories(geoIpConfigDir); Files.createDirectories(geoIpConfigDir);
Files.copy(new ByteArrayInputStream(StreamsUtils.copyToBytesFromClasspath("/GeoLite2-City.mmdb.gz")), Files.copy(new ByteArrayInputStream(StreamsUtils.copyToBytesFromClasspath("/GeoLite2-City.mmdb")),
geoIpConfigDir.resolve("GeoLite2-City.mmdb.gz")); geoIpConfigDir.resolve("GeoLite2-City.mmdb"));
Files.copy(new ByteArrayInputStream(StreamsUtils.copyToBytesFromClasspath("/GeoLite2-Country.mmdb.gz")), Files.copy(new ByteArrayInputStream(StreamsUtils.copyToBytesFromClasspath("/GeoLite2-Country.mmdb")),
geoIpConfigDir.resolve("GeoLite2-Country.mmdb.gz")); geoIpConfigDir.resolve("GeoLite2-Country.mmdb"));
Files.copy(new ByteArrayInputStream(StreamsUtils.copyToBytesFromClasspath("/GeoLite2-ASN.mmdb.gz")), Files.copy(new ByteArrayInputStream(StreamsUtils.copyToBytesFromClasspath("/GeoLite2-ASN.mmdb")),
geoIpConfigDir.resolve("GeoLite2-ASN.mmdb.gz")); geoIpConfigDir.resolve("GeoLite2-ASN.mmdb"));
NodeCache cache = randomFrom(NoCache.getInstance(), new GeoIpCache(randomNonNegativeLong())); NodeCache cache = randomFrom(NoCache.getInstance(), new GeoIpCache(randomNonNegativeLong()));
databaseReaders = IngestGeoIpPlugin.loadDatabaseReaders(geoIpConfigDir, cache); databaseReaders = IngestGeoIpPlugin.loadDatabaseReaders(geoIpConfigDir, cache);
@ -111,7 +111,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
Map<String, Object> config = new HashMap<>(); Map<String, Object> config = new HashMap<>();
config.put("field", "_field"); config.put("field", "_field");
config.put("database_file", "GeoLite2-Country.mmdb.gz"); config.put("database_file", "GeoLite2-Country.mmdb");
String processorTag = randomAlphaOfLength(10); String processorTag = randomAlphaOfLength(10);
GeoIpProcessor processor = factory.create(null, processorTag, config); GeoIpProcessor processor = factory.create(null, processorTag, config);
@ -129,7 +129,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
Map<String, Object> config = new HashMap<>(); Map<String, Object> config = new HashMap<>();
config.put("field", "_field"); config.put("field", "_field");
config.put("database_file", "GeoLite2-ASN.mmdb.gz"); config.put("database_file", "GeoLite2-ASN.mmdb");
String processorTag = randomAlphaOfLength(10); String processorTag = randomAlphaOfLength(10);
GeoIpProcessor processor = factory.create(null, processorTag, config); GeoIpProcessor processor = factory.create(null, processorTag, config);
@ -157,7 +157,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseReaders); GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseReaders);
Map<String, Object> config = new HashMap<>(); Map<String, Object> config = new HashMap<>();
config.put("field", "_field"); config.put("field", "_field");
config.put("database_file", "GeoLite2-Country.mmdb.gz"); config.put("database_file", "GeoLite2-Country.mmdb");
GeoIpProcessor processor = factory.create(null, null, config); GeoIpProcessor processor = factory.create(null, null, config);
assertThat(processor.getField(), equalTo("_field")); assertThat(processor.getField(), equalTo("_field"));
assertThat(processor.getTargetField(), equalTo("geoip")); assertThat(processor.getTargetField(), equalTo("geoip"));
@ -170,7 +170,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseReaders); GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseReaders);
Map<String, Object> config = new HashMap<>(); Map<String, Object> config = new HashMap<>();
config.put("field", "_field"); config.put("field", "_field");
config.put("database_file", "GeoLite2-Country.mmdb.gz"); config.put("database_file", "GeoLite2-Country.mmdb");
EnumSet<GeoIpProcessor.Property> asnOnlyProperties = EnumSet.copyOf(GeoIpProcessor.Property.ALL_ASN_PROPERTIES); EnumSet<GeoIpProcessor.Property> asnOnlyProperties = EnumSet.copyOf(GeoIpProcessor.Property.ALL_ASN_PROPERTIES);
asnOnlyProperties.remove(GeoIpProcessor.Property.IP); asnOnlyProperties.remove(GeoIpProcessor.Property.IP);
String asnProperty = RandomPicks.randomFrom(Randomness.get(), asnOnlyProperties).toString(); String asnProperty = RandomPicks.randomFrom(Randomness.get(), asnOnlyProperties).toString();
@ -184,7 +184,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseReaders); GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseReaders);
Map<String, Object> config = new HashMap<>(); Map<String, Object> config = new HashMap<>();
config.put("field", "_field"); config.put("field", "_field");
config.put("database_file", "GeoLite2-ASN.mmdb.gz"); config.put("database_file", "GeoLite2-ASN.mmdb");
EnumSet<GeoIpProcessor.Property> cityOnlyProperties = EnumSet.copyOf(GeoIpProcessor.Property.ALL_CITY_PROPERTIES); EnumSet<GeoIpProcessor.Property> cityOnlyProperties = EnumSet.copyOf(GeoIpProcessor.Property.ALL_CITY_PROPERTIES);
cityOnlyProperties.remove(GeoIpProcessor.Property.IP); cityOnlyProperties.remove(GeoIpProcessor.Property.IP);
String cityProperty = RandomPicks.randomFrom(Randomness.get(), cityOnlyProperties).toString(); String cityProperty = RandomPicks.randomFrom(Randomness.get(), cityOnlyProperties).toString();
@ -199,9 +199,9 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
Map<String, Object> config = new HashMap<>(); Map<String, Object> config = new HashMap<>();
config.put("field", "_field"); config.put("field", "_field");
config.put("database_file", "does-not-exist.mmdb.gz"); config.put("database_file", "does-not-exist.mmdb");
Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, config)); Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, config));
assertThat(e.getMessage(), equalTo("[database_file] database file [does-not-exist.mmdb.gz] doesn't exist")); assertThat(e.getMessage(), equalTo("[database_file] database file [does-not-exist.mmdb] doesn't exist"));
} }
public void testBuildFields() throws Exception { public void testBuildFields() throws Exception {
@ -249,12 +249,12 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
Path configDir = createTempDir(); Path configDir = createTempDir();
Path geoIpConfigDir = configDir.resolve("ingest-geoip"); Path geoIpConfigDir = configDir.resolve("ingest-geoip");
Files.createDirectories(geoIpConfigDir); Files.createDirectories(geoIpConfigDir);
Files.copy(new ByteArrayInputStream(StreamsUtils.copyToBytesFromClasspath("/GeoLite2-City.mmdb.gz")), Files.copy(new ByteArrayInputStream(StreamsUtils.copyToBytesFromClasspath("/GeoLite2-City.mmdb")),
geoIpConfigDir.resolve("GeoLite2-City.mmdb.gz")); geoIpConfigDir.resolve("GeoLite2-City.mmdb"));
Files.copy(new ByteArrayInputStream(StreamsUtils.copyToBytesFromClasspath("/GeoLite2-Country.mmdb.gz")), Files.copy(new ByteArrayInputStream(StreamsUtils.copyToBytesFromClasspath("/GeoLite2-Country.mmdb")),
geoIpConfigDir.resolve("GeoLite2-Country.mmdb.gz")); geoIpConfigDir.resolve("GeoLite2-Country.mmdb"));
Files.copy(new ByteArrayInputStream(StreamsUtils.copyToBytesFromClasspath("/GeoLite2-ASN.mmdb.gz")), Files.copy(new ByteArrayInputStream(StreamsUtils.copyToBytesFromClasspath("/GeoLite2-ASN.mmdb")),
geoIpConfigDir.resolve("GeoLite2-ASN.mmdb.gz")); geoIpConfigDir.resolve("GeoLite2-ASN.mmdb"));
// Loading another database reader instances, because otherwise we can't test lazy loading as the // Loading another database reader instances, because otherwise we can't test lazy loading as the
// database readers used at class level are reused between tests. (we want to keep that otherwise running this // database readers used at class level are reused between tests. (we want to keep that otherwise running this
@ -268,15 +268,15 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
Map<String, Object> config = new HashMap<>(); Map<String, Object> config = new HashMap<>();
config.put("field", "_field"); config.put("field", "_field");
config.put("database_file", "GeoLite2-City.mmdb.gz"); config.put("database_file", "GeoLite2-City.mmdb");
factory.create(null, "_tag", config); factory.create(null, "_tag", config);
config = new HashMap<>(); config = new HashMap<>();
config.put("field", "_field"); config.put("field", "_field");
config.put("database_file", "GeoLite2-Country.mmdb.gz"); config.put("database_file", "GeoLite2-Country.mmdb");
factory.create(null, "_tag", config); factory.create(null, "_tag", config);
config = new HashMap<>(); config = new HashMap<>();
config.put("field", "_field"); config.put("field", "_field");
config.put("database_file", "GeoLite2-ASN.mmdb.gz"); config.put("database_file", "GeoLite2-ASN.mmdb");
factory.create(null, "_tag", config); factory.create(null, "_tag", config);
for (DatabaseReaderLazyLoader lazyLoader : databaseReaders.values()) { for (DatabaseReaderLazyLoader lazyLoader : databaseReaders.values()) {

30
plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java
View File

@ -24,13 +24,11 @@ import org.elasticsearch.ingest.RandomDocumentPicks;
import org.elasticsearch.ingest.IngestDocument; import org.elasticsearch.ingest.IngestDocument;
import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.ESTestCase;
import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.util.Collections; import java.util.Collections;
import java.util.EnumSet; import java.util.EnumSet;
import java.util.HashMap; import java.util.HashMap;
import java.util.Map; import java.util.Map;
import java.util.zip.GZIPInputStream;
import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument; import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument;
import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.containsString;
@ -40,7 +38,7 @@ import static org.hamcrest.Matchers.is;
public class GeoIpProcessorTests extends ESTestCase { public class GeoIpProcessorTests extends ESTestCase {
public void testCity() throws Exception { public void testCity() throws Exception {
InputStream database = getDatabaseFileInputStream("/GeoLite2-City.mmdb.gz"); InputStream database = getDatabaseFileInputStream("/GeoLite2-City.mmdb");
GeoIpProcessor processor = new GeoIpProcessor(randomAlphaOfLength(10), "source_field", GeoIpProcessor processor = new GeoIpProcessor(randomAlphaOfLength(10), "source_field",
new DatabaseReader.Builder(database).build(), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class), false); new DatabaseReader.Builder(database).build(), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class), false);
@ -64,7 +62,7 @@ public class GeoIpProcessorTests extends ESTestCase {
} }
public void testNullValueWithIgnoreMissing() throws Exception { public void testNullValueWithIgnoreMissing() throws Exception {
InputStream database = getDatabaseFileInputStream("/GeoLite2-City.mmdb.gz"); InputStream database = getDatabaseFileInputStream("/GeoLite2-City.mmdb");
GeoIpProcessor processor = new GeoIpProcessor(randomAlphaOfLength(10), "source_field", GeoIpProcessor processor = new GeoIpProcessor(randomAlphaOfLength(10), "source_field",
new DatabaseReader.Builder(database).build(), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class), true); new DatabaseReader.Builder(database).build(), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class), true);
IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(),
@ -75,7 +73,7 @@ public class GeoIpProcessorTests extends ESTestCase {
} }
public void testNonExistentWithIgnoreMissing() throws Exception { public void testNonExistentWithIgnoreMissing() throws Exception {
InputStream database = getDatabaseFileInputStream("/GeoLite2-City.mmdb.gz"); InputStream database = getDatabaseFileInputStream("/GeoLite2-City.mmdb");
GeoIpProcessor processor = new GeoIpProcessor(randomAlphaOfLength(10), "source_field", GeoIpProcessor processor = new GeoIpProcessor(randomAlphaOfLength(10), "source_field",
new DatabaseReader.Builder(database).build(), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class), true); new DatabaseReader.Builder(database).build(), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class), true);
IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.emptyMap()); IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.emptyMap());
@ -85,7 +83,7 @@ public class GeoIpProcessorTests extends ESTestCase {
} }
public void testNullWithoutIgnoreMissing() throws Exception { public void testNullWithoutIgnoreMissing() throws Exception {
InputStream database = getDatabaseFileInputStream("/GeoLite2-City.mmdb.gz"); InputStream database = getDatabaseFileInputStream("/GeoLite2-City.mmdb");
GeoIpProcessor processor = new GeoIpProcessor(randomAlphaOfLength(10), "source_field", GeoIpProcessor processor = new GeoIpProcessor(randomAlphaOfLength(10), "source_field",
new DatabaseReader.Builder(database).build(), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class), false); new DatabaseReader.Builder(database).build(), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class), false);
IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(),
@ -96,7 +94,7 @@ public class GeoIpProcessorTests extends ESTestCase {
} }
public void testNonExistentWithoutIgnoreMissing() throws Exception { public void testNonExistentWithoutIgnoreMissing() throws Exception {
InputStream database = getDatabaseFileInputStream("/GeoLite2-City.mmdb.gz"); InputStream database = getDatabaseFileInputStream("/GeoLite2-City.mmdb");
GeoIpProcessor processor = new GeoIpProcessor(randomAlphaOfLength(10), "source_field", GeoIpProcessor processor = new GeoIpProcessor(randomAlphaOfLength(10), "source_field",
new DatabaseReader.Builder(database).build(), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class), false); new DatabaseReader.Builder(database).build(), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class), false);
IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.emptyMap()); IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.emptyMap());
@ -106,7 +104,7 @@ public class GeoIpProcessorTests extends ESTestCase {
} }
public void testCity_withIpV6() throws Exception { public void testCity_withIpV6() throws Exception {
InputStream database = getDatabaseFileInputStream("/GeoLite2-City.mmdb.gz"); InputStream database = getDatabaseFileInputStream("/GeoLite2-City.mmdb");
GeoIpProcessor processor = new GeoIpProcessor(randomAlphaOfLength(10), "source_field", GeoIpProcessor processor = new GeoIpProcessor(randomAlphaOfLength(10), "source_field",
new DatabaseReader.Builder(database).build(), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class), false); new DatabaseReader.Builder(database).build(), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class), false);
@ -134,7 +132,7 @@ public class GeoIpProcessorTests extends ESTestCase {
} }
public void testCityWithMissingLocation() throws Exception { public void testCityWithMissingLocation() throws Exception {
InputStream database = getDatabaseFileInputStream("/GeoLite2-City.mmdb.gz"); InputStream database = getDatabaseFileInputStream("/GeoLite2-City.mmdb");
GeoIpProcessor processor = new GeoIpProcessor(randomAlphaOfLength(10), "source_field", GeoIpProcessor processor = new GeoIpProcessor(randomAlphaOfLength(10), "source_field",
new DatabaseReader.Builder(database).build(), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class), false); new DatabaseReader.Builder(database).build(), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class), false);
@ -151,7 +149,7 @@ public class GeoIpProcessorTests extends ESTestCase {
} }
public void testCountry() throws Exception { public void testCountry() throws Exception {
InputStream database = getDatabaseFileInputStream("/GeoLite2-Country.mmdb.gz"); InputStream database = getDatabaseFileInputStream("/GeoLite2-Country.mmdb");
GeoIpProcessor processor = new GeoIpProcessor(randomAlphaOfLength(10), "source_field", GeoIpProcessor processor = new GeoIpProcessor(randomAlphaOfLength(10), "source_field",
new DatabaseReader.Builder(database).build(), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class), false); new DatabaseReader.Builder(database).build(), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class), false);
@ -171,7 +169,7 @@ public class GeoIpProcessorTests extends ESTestCase {
} }
public void testCountryWithMissingLocation() throws Exception { public void testCountryWithMissingLocation() throws Exception {
InputStream database = getDatabaseFileInputStream("/GeoLite2-Country.mmdb.gz"); InputStream database = getDatabaseFileInputStream("/GeoLite2-Country.mmdb");
GeoIpProcessor processor = new GeoIpProcessor(randomAlphaOfLength(10), "source_field", GeoIpProcessor processor = new GeoIpProcessor(randomAlphaOfLength(10), "source_field",
new DatabaseReader.Builder(database).build(), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class), false); new DatabaseReader.Builder(database).build(), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class), false);
@ -189,7 +187,7 @@ public class GeoIpProcessorTests extends ESTestCase {
public void testAsn() throws Exception { public void testAsn() throws Exception {
String ip = "82.170.213.79"; String ip = "82.170.213.79";
InputStream database = getDatabaseFileInputStream("/GeoLite2-ASN.mmdb.gz"); InputStream database = getDatabaseFileInputStream("/GeoLite2-ASN.mmdb");
GeoIpProcessor processor = new GeoIpProcessor(randomAlphaOfLength(10), "source_field", GeoIpProcessor processor = new GeoIpProcessor(randomAlphaOfLength(10), "source_field",
new DatabaseReader.Builder(database).build(), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class), false); new DatabaseReader.Builder(database).build(), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class), false);
@ -208,7 +206,7 @@ public class GeoIpProcessorTests extends ESTestCase {
} }
public void testAddressIsNotInTheDatabase() throws Exception { public void testAddressIsNotInTheDatabase() throws Exception {
InputStream database = getDatabaseFileInputStream("/GeoLite2-City.mmdb.gz"); InputStream database = getDatabaseFileInputStream("/GeoLite2-City.mmdb");
GeoIpProcessor processor = new GeoIpProcessor(randomAlphaOfLength(10), "source_field", GeoIpProcessor processor = new GeoIpProcessor(randomAlphaOfLength(10), "source_field",
new DatabaseReader.Builder(database).build(), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class), false); new DatabaseReader.Builder(database).build(), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class), false);
@ -221,7 +219,7 @@ public class GeoIpProcessorTests extends ESTestCase {
/** Don't silently do DNS lookups or anything trappy on bogus data */ /** Don't silently do DNS lookups or anything trappy on bogus data */
public void testInvalid() throws Exception { public void testInvalid() throws Exception {
InputStream database = getDatabaseFileInputStream("/GeoLite2-City.mmdb.gz"); InputStream database = getDatabaseFileInputStream("/GeoLite2-City.mmdb");
GeoIpProcessor processor = new GeoIpProcessor(randomAlphaOfLength(10), "source_field", GeoIpProcessor processor = new GeoIpProcessor(randomAlphaOfLength(10), "source_field",
new DatabaseReader.Builder(database).build(), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class), false); new DatabaseReader.Builder(database).build(), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class), false);
@ -232,8 +230,8 @@ public class GeoIpProcessorTests extends ESTestCase {
assertThat(e.getMessage(), containsString("not an IP string literal")); assertThat(e.getMessage(), containsString("not an IP string literal"));
} }
private static InputStream getDatabaseFileInputStream(String path) throws IOException { private static InputStream getDatabaseFileInputStream(String path) {
return new GZIPInputStream(GeoIpProcessor.class.getResourceAsStream(path)); return GeoIpProcessor.class.getResourceAsStream(path);
} }
} }

4
plugins/ingest-geoip/src/test/resources/rest-api-spec/test/ingest_geoip/20_geoip_processor.yml
View File

@ -98,7 +98,7 @@
{ {
"geoip" : { "geoip" : {
"field" : "field1", "field" : "field1",
"database_file" : "GeoLite2-Country.mmdb.gz" "database_file" : "GeoLite2-Country.mmdb"
} }
} }
] ]
@ -208,7 +208,7 @@
{ {
"geoip" : { "geoip" : {
"field" : "field1", "field" : "field1",
"database_file" : "GeoLite2-ASN.mmdb.gz" "database_file" : "GeoLite2-ASN.mmdb"
} }
} }
] ]