Update ingest useragent plugin to use new ingest plugin

This commit is contained in:
Ryan Ernst 2016-07-01 14:16:09 -07:00
parent 40e1fe7d9e
commit 10261a615b
4 changed files with 35 additions and 20 deletions

View File

@ -48,7 +48,7 @@ public class IngestGeoIpPlugin extends Plugin implements IngestPlugin, Closeable
private Map<String, DatabaseReader> databaseReaders; private Map<String, DatabaseReader> databaseReaders;
@Override @Override
public synchronized Map<String, Processor.Factory> getProcessors( public Map<String, Processor.Factory> getProcessors(
Environment env, ScriptService scriptService, TemplateService templateService) { Environment env, ScriptService scriptService, TemplateService templateService) {
if (databaseReaders != null) { if (databaseReaders != null) {
throw new IllegalStateException("called onModule twice for geoip plugin!!"); throw new IllegalStateException("called onModule twice for geoip plugin!!");

View File

@ -20,8 +20,14 @@
package org.elasticsearch.ingest.useragent; package org.elasticsearch.ingest.useragent;
import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.env.Environment;
import org.elasticsearch.ingest.Processor;
import org.elasticsearch.ingest.TemplateService;
import org.elasticsearch.node.NodeModule; import org.elasticsearch.node.NodeModule;
import org.elasticsearch.plugins.IngestPlugin;
import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.script.ScriptService;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
@ -34,28 +40,36 @@ import java.util.HashMap;
import java.util.Map; import java.util.Map;
import java.util.stream.Stream; import java.util.stream.Stream;
public class IngestUserAgentPlugin extends Plugin { public class IngestUserAgentPlugin extends Plugin implements IngestPlugin {
private final Setting<Long> CACHE_SIZE_SETTING = Setting.longSetting("ingest.useragent.cache_size", 1000, 0, private final Setting<Long> CACHE_SIZE_SETTING = Setting.longSetting("ingest.useragent.cache_size", 1000, 0,
Setting.Property.NodeScope); Setting.Property.NodeScope);
static final String DEFAULT_PARSER_NAME = "_default_"; static final String DEFAULT_PARSER_NAME = "_default_";
private final Settings settings;
public void onModule(NodeModule nodeModule) throws IOException { IngestUserAgentPlugin(Settings settings) {
Path userAgentConfigDirectory = nodeModule.getNode().getEnvironment().configFile().resolve("ingest-useragent"); this.settings = settings;
}
@Override
public Map<String, Processor.Factory> getProcessors(
Environment env, ScriptService scriptService, TemplateService templateService) {
Path userAgentConfigDirectory = env.configFile().resolve("ingest-useragent");
if (Files.exists(userAgentConfigDirectory) == false && Files.isDirectory(userAgentConfigDirectory)) { if (Files.exists(userAgentConfigDirectory) == false && Files.isDirectory(userAgentConfigDirectory)) {
throw new IllegalStateException( throw new IllegalStateException(
"the user agent directory [" + userAgentConfigDirectory + "] containing the regex file doesn't exist"); "the user agent directory [" + userAgentConfigDirectory + "] containing the regex file doesn't exist");
} }
long cacheSize = CACHE_SIZE_SETTING.get(nodeModule.getNode().settings()); long cacheSize = CACHE_SIZE_SETTING.get(settings);
Map<String, UserAgentParser> userAgentParsers;
UserAgentCache cache = new UserAgentCache(cacheSize); try {
userAgentParsers = createUserAgentParsers(userAgentConfigDirectory, new UserAgentCache(cacheSize));
Map<String, UserAgentParser> userAgentParsers = createUserAgentParsers(userAgentConfigDirectory, cache); } catch (IOException e) {
throw new RuntimeException(e);
nodeModule.registerProcessor(UserAgentProcessor.TYPE, (registry) -> new UserAgentProcessor.Factory(userAgentParsers)); }
return Collections.singletonMap(UserAgentProcessor.TYPE, new UserAgentProcessor.Factory(userAgentParsers));
} }
static Map<String, UserAgentParser> createUserAgentParsers(Path userAgentConfigDirectory, UserAgentCache cache) throws IOException { static Map<String, UserAgentParser> createUserAgentParsers(Path userAgentConfigDirectory, UserAgentCache cache) throws IOException {

View File

@ -194,7 +194,8 @@ public class UserAgentProcessor extends AbstractProcessor {
} }
@Override @Override
public UserAgentProcessor create(String processorTag, Map<String, Object> config) throws Exception { public UserAgentProcessor create(Map<String, Processor.Factory> factories, String processorTag,
Map<String, Object> config) throws Exception {
String field = readStringProperty(TYPE, processorTag, config, "field"); String field = readStringProperty(TYPE, processorTag, config, "field");
String targetField = readStringProperty(TYPE, processorTag, config, "target_field", "useragent"); String targetField = readStringProperty(TYPE, processorTag, config, "target_field", "useragent");
String regexFilename = readStringProperty(TYPE, processorTag, config, "regex_file", IngestUserAgentPlugin.DEFAULT_PARSER_NAME); String regexFilename = readStringProperty(TYPE, processorTag, config, "regex_file", IngestUserAgentPlugin.DEFAULT_PARSER_NAME);

View File

@ -81,7 +81,7 @@ public class UserAgentProcessorFactoryTests extends ESTestCase {
String processorTag = randomAsciiOfLength(10); String processorTag = randomAsciiOfLength(10);
UserAgentProcessor processor = factory.create(processorTag, config); UserAgentProcessor processor = factory.create(null, processorTag, config);
assertThat(processor.getTag(), equalTo(processorTag)); assertThat(processor.getTag(), equalTo(processorTag));
assertThat(processor.getField(), equalTo("_field")); assertThat(processor.getField(), equalTo("_field"));
assertThat(processor.getTargetField(), equalTo("useragent")); assertThat(processor.getTargetField(), equalTo("useragent"));
@ -98,7 +98,7 @@ public class UserAgentProcessorFactoryTests extends ESTestCase {
config.put("field", "_field"); config.put("field", "_field");
config.put("target_field", "_target_field"); config.put("target_field", "_target_field");
UserAgentProcessor processor = factory.create(null, config); UserAgentProcessor processor = factory.create(null, null, config);
assertThat(processor.getField(), equalTo("_field")); assertThat(processor.getField(), equalTo("_field"));
assertThat(processor.getTargetField(), equalTo("_target_field")); assertThat(processor.getTargetField(), equalTo("_target_field"));
} }
@ -110,7 +110,7 @@ public class UserAgentProcessorFactoryTests extends ESTestCase {
config.put("field", "_field"); config.put("field", "_field");
config.put("regex_file", regexWithoutDevicesFilename); config.put("regex_file", regexWithoutDevicesFilename);
UserAgentProcessor processor = factory.create(null, config); UserAgentProcessor processor = factory.create(null, null, config);
assertThat(processor.getField(), equalTo("_field")); assertThat(processor.getField(), equalTo("_field"));
assertThat(processor.getUaParser().getUaPatterns().size(), greaterThan(0)); assertThat(processor.getUaParser().getUaPatterns().size(), greaterThan(0));
assertThat(processor.getUaParser().getOsPatterns().size(), greaterThan(0)); assertThat(processor.getUaParser().getOsPatterns().size(), greaterThan(0));
@ -124,7 +124,7 @@ public class UserAgentProcessorFactoryTests extends ESTestCase {
config.put("field", "_field"); config.put("field", "_field");
config.put("regex_file", "does-not-exist.yaml"); config.put("regex_file", "does-not-exist.yaml");
ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, config)); ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, config));
assertThat(e.getMessage(), equalTo("[regex_file] regex file [does-not-exist.yaml] doesn't exist (has to exist at node startup)")); assertThat(e.getMessage(), equalTo("[regex_file] regex file [does-not-exist.yaml] doesn't exist (has to exist at node startup)"));
} }
@ -144,7 +144,7 @@ public class UserAgentProcessorFactoryTests extends ESTestCase {
config.put("field", "_field"); config.put("field", "_field");
config.put("properties", fieldNames); config.put("properties", fieldNames);
UserAgentProcessor processor = factory.create(null, config); UserAgentProcessor processor = factory.create(null, null, config);
assertThat(processor.getField(), equalTo("_field")); assertThat(processor.getField(), equalTo("_field"));
assertThat(processor.getProperties(), equalTo(properties)); assertThat(processor.getProperties(), equalTo(properties));
} }
@ -156,7 +156,7 @@ public class UserAgentProcessorFactoryTests extends ESTestCase {
config.put("field", "_field"); config.put("field", "_field");
config.put("properties", Collections.singletonList("invalid")); config.put("properties", Collections.singletonList("invalid"));
ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, config)); ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, config));
assertThat(e.getMessage(), equalTo("[properties] illegal property value [invalid]. valid values are [NAME, MAJOR, MINOR, " assertThat(e.getMessage(), equalTo("[properties] illegal property value [invalid]. valid values are [NAME, MAJOR, MINOR, "
+ "PATCH, OS, OS_NAME, OS_MAJOR, OS_MINOR, DEVICE, BUILD]")); + "PATCH, OS, OS_NAME, OS_MAJOR, OS_MINOR, DEVICE, BUILD]"));
} }
@ -168,7 +168,7 @@ public class UserAgentProcessorFactoryTests extends ESTestCase {
config.put("field", "_field"); config.put("field", "_field");
config.put("properties", "invalid"); config.put("properties", "invalid");
ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, config)); ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, config));
assertThat(e.getMessage(), equalTo("[properties] property isn't a list, but of type [java.lang.String]")); assertThat(e.getMessage(), equalTo("[properties] property isn't a list, but of type [java.lang.String]"));
} }
} }