diff --git a/core/src/main/java/org/elasticsearch/plugins/PluginManager.java b/core/src/main/java/org/elasticsearch/plugins/PluginManager.java index 7cd50409fb6..56da7176e9d 100644 --- a/core/src/main/java/org/elasticsearch/plugins/PluginManager.java +++ b/core/src/main/java/org/elasticsearch/plugins/PluginManager.java @@ -101,6 +101,7 @@ public class PluginManager { "discovery-ec2", "discovery-gce", "discovery-multicast", + "ingest-geoip", "lang-javascript", "lang-plan-a", "lang-python", diff --git a/core/src/main/resources/org/elasticsearch/plugins/plugin-install.help b/core/src/main/resources/org/elasticsearch/plugins/plugin-install.help index 8c73e3837a4..d46f7dca29a 100644 --- a/core/src/main/resources/org/elasticsearch/plugins/plugin-install.help +++ b/core/src/main/resources/org/elasticsearch/plugins/plugin-install.help @@ -43,6 +43,7 @@ OFFICIAL PLUGINS - discovery-ec2 - discovery-gce - discovery-multicast + - ingest-geoip - lang-javascript - lang-plan-a - lang-python diff --git a/modules/ingest-grok/src/main/java/org/elasticsearch/ingest/grok/GrokProcessor.java b/modules/ingest-grok/src/main/java/org/elasticsearch/ingest/grok/GrokProcessor.java index 2c63646ff81..56c04f09709 100644 --- a/modules/ingest-grok/src/main/java/org/elasticsearch/ingest/grok/GrokProcessor.java +++ b/modules/ingest-grok/src/main/java/org/elasticsearch/ingest/grok/GrokProcessor.java @@ -19,16 +19,10 @@ package org.elasticsearch.ingest.grok; -import org.elasticsearch.ingest.core.IngestDocument; import org.elasticsearch.ingest.core.ConfigurationUtils; +import org.elasticsearch.ingest.core.IngestDocument; import org.elasticsearch.ingest.core.Processor; -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.nio.charset.StandardCharsets; -import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -70,47 +64,17 @@ public final class GrokProcessor implements Processor { public final static class Factory implements Processor.Factory { - private final static String[] PATTERN_NAMES = new String[] { - "aws", "bacula", "bro", "exim", "firewalls", "grok-patterns", "haproxy", - "java", "junos", "linux-syslog", "mcollective-patterns", "mongodb", "nagios", - "postgresql", "rails", "redis", "ruby" - }; - private final Map builtinPatternBank; + private final Map builtinPatterns; - public Factory() throws IOException { - // TODO(simonw): we should have a static helper method to load these patterns and make this - // factory only accept a String->String map instead. That way we can load - // the patterns in the IngestGrokPlugin ctor or even in a static context and this ctor doesn't need to throw any exception. - Map builtinPatterns = new HashMap<>(); - for (String pattern : PATTERN_NAMES) { - try(InputStream is = getClass().getResourceAsStream("/patterns/" + pattern)) { - loadBankFromStream(builtinPatterns, is); - } - } - this.builtinPatternBank = Collections.unmodifiableMap(builtinPatterns); - } - - static void loadBankFromStream(Map patternBank, InputStream inputStream) throws IOException { - String line; - BufferedReader br = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8)); - while ((line = br.readLine()) != null) { - String trimmedLine = line.replaceAll("^\\s+", ""); - if (trimmedLine.startsWith("#") || trimmedLine.length() == 0) { - continue; - } - - String[] parts = trimmedLine.split("\\s+", 2); - if (parts.length == 2) { - patternBank.put(parts[0], parts[1]); - } - } + public Factory(Map builtinPatterns) { + this.builtinPatterns = builtinPatterns; } public GrokProcessor create(Map config) throws Exception { String matchField = ConfigurationUtils.readStringProperty(config, "field"); String matchPattern = ConfigurationUtils.readStringProperty(config, "pattern"); Map customPatternBank = ConfigurationUtils.readOptionalMap(config, "pattern_definitions"); - Map patternBank = new HashMap<>(builtinPatternBank); + Map patternBank = new HashMap<>(builtinPatterns); if (customPatternBank != null) { patternBank.putAll(customPatternBank); } diff --git a/modules/ingest-grok/src/main/java/org/elasticsearch/ingest/grok/IngestGrokPlugin.java b/modules/ingest-grok/src/main/java/org/elasticsearch/ingest/grok/IngestGrokPlugin.java index 2b61b5e9073..f6423fe40cf 100644 --- a/modules/ingest-grok/src/main/java/org/elasticsearch/ingest/grok/IngestGrokPlugin.java +++ b/modules/ingest-grok/src/main/java/org/elasticsearch/ingest/grok/IngestGrokPlugin.java @@ -22,10 +22,29 @@ package org.elasticsearch.ingest.grok; import org.elasticsearch.ingest.IngestModule; import org.elasticsearch.plugins.Plugin; +import java.io.BufferedReader; import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; public class IngestGrokPlugin extends Plugin { + private static final String[] PATTERN_NAMES = new String[] { + "aws", "bacula", "bro", "exim", "firewalls", "grok-patterns", "haproxy", + "java", "junos", "linux-syslog", "mcollective-patterns", "mongodb", "nagios", + "postgresql", "rails", "redis", "ruby" + }; + + private final Map builtinPatterns; + + public IngestGrokPlugin() throws IOException { + this.builtinPatterns = loadBuiltinPatterns(); + } + @Override public String name() { return "ingest-grok"; @@ -37,12 +56,32 @@ public class IngestGrokPlugin extends Plugin { } public void onModule(IngestModule ingestModule) { - ingestModule.registerProcessor(GrokProcessor.TYPE, (environment, templateService) -> { - try { - return new GrokProcessor.Factory(); - } catch (IOException e) { - throw new RuntimeException(e); + ingestModule.registerProcessor(GrokProcessor.TYPE, (environment, templateService) -> new GrokProcessor.Factory(builtinPatterns)); + } + + static Map loadBuiltinPatterns() throws IOException { + Map builtinPatterns = new HashMap<>(); + for (String pattern : PATTERN_NAMES) { + try(InputStream is = IngestGrokPlugin.class.getResourceAsStream("/patterns/" + pattern)) { + loadPatterns(builtinPatterns, is); } - }); + } + return Collections.unmodifiableMap(builtinPatterns); + } + + private static void loadPatterns(Map patternBank, InputStream inputStream) throws IOException { + String line; + BufferedReader br = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8)); + while ((line = br.readLine()) != null) { + String trimmedLine = line.replaceAll("^\\s+", ""); + if (trimmedLine.startsWith("#") || trimmedLine.length() == 0) { + continue; + } + + String[] parts = trimmedLine.split("\\s+", 2); + if (parts.length == 2) { + patternBank.put(parts[0], parts[1]); + } + } } } diff --git a/modules/ingest-grok/src/test/java/org/elasticsearch/ingest/grok/GrokProcessorFactoryTests.java b/modules/ingest-grok/src/test/java/org/elasticsearch/ingest/grok/GrokProcessorFactoryTests.java index 11c1024eb82..419e48874cd 100644 --- a/modules/ingest-grok/src/test/java/org/elasticsearch/ingest/grok/GrokProcessorFactoryTests.java +++ b/modules/ingest-grok/src/test/java/org/elasticsearch/ingest/grok/GrokProcessorFactoryTests.java @@ -19,7 +19,6 @@ package org.elasticsearch.ingest.grok; -import org.elasticsearch.ingest.grok.GrokProcessor; import org.elasticsearch.test.ESTestCase; import java.util.Collections; @@ -32,7 +31,7 @@ import static org.hamcrest.Matchers.notNullValue; public class GrokProcessorFactoryTests extends ESTestCase { public void testBuild() throws Exception { - GrokProcessor.Factory factory = new GrokProcessor.Factory(); + GrokProcessor.Factory factory = new GrokProcessor.Factory(Collections.emptyMap()); Map config = new HashMap<>(); config.put("field", "_field"); @@ -43,7 +42,7 @@ public class GrokProcessorFactoryTests extends ESTestCase { } public void testCreateWithCustomPatterns() throws Exception { - GrokProcessor.Factory factory = new GrokProcessor.Factory(); + GrokProcessor.Factory factory = new GrokProcessor.Factory(Collections.emptyMap()); Map config = new HashMap<>(); config.put("field", "_field"); diff --git a/modules/ingest-grok/src/test/java/org/elasticsearch/ingest/grok/GrokTests.java b/modules/ingest-grok/src/test/java/org/elasticsearch/ingest/grok/GrokTests.java index 89e44fc7239..21ca17ad713 100644 --- a/modules/ingest-grok/src/test/java/org/elasticsearch/ingest/grok/GrokTests.java +++ b/modules/ingest-grok/src/test/java/org/elasticsearch/ingest/grok/GrokTests.java @@ -19,13 +19,10 @@ package org.elasticsearch.ingest.grok; -import org.elasticsearch.ingest.grok.Grok; -import org.elasticsearch.ingest.grok.GrokProcessor; import org.elasticsearch.test.ESTestCase; import org.junit.Before; import java.io.IOException; -import java.io.InputStream; import java.util.Arrays; import java.util.HashMap; import java.util.List; @@ -39,22 +36,9 @@ import static org.hamcrest.Matchers.nullValue; public class GrokTests extends ESTestCase { private Map basePatterns; - private Map newBankFromStreams(InputStream... inputStreams) throws IOException { - Map patternBank = new HashMap<>(); - - for (InputStream is : inputStreams) { - GrokProcessor.Factory.loadBankFromStream(patternBank, is); - } - - return patternBank; - } - @Before public void setup() throws IOException { - basePatterns = newBankFromStreams( - getClass().getResourceAsStream("/patterns/grok-patterns"), - getClass().getResourceAsStream("/patterns/linux-syslog") - ); + basePatterns = IngestGrokPlugin.loadBuiltinPatterns(); } public void testMatchWithoutCaptures() { diff --git a/settings.gradle b/settings.gradle index 39e0b4fb5b0..63e78dea047 100644 --- a/settings.gradle +++ b/settings.gradle @@ -11,10 +11,10 @@ List projects = [ 'test:framework', 'test:fixtures:example-fixture', 'test:fixtures:hdfs-fixture', + 'modules:ingest-grok', 'modules:lang-expression', 'modules:lang-groovy', 'modules:lang-mustache', - 'modules:ingest-grok', 'plugins:analysis-icu', 'plugins:analysis-kuromoji', 'plugins:analysis-phonetic',