Update ingest useragent plugin to use new ingest plugin
This commit is contained in:
parent
40e1fe7d9e
commit
10261a615b
|
@ -48,7 +48,7 @@ public class IngestGeoIpPlugin extends Plugin implements IngestPlugin, Closeable
|
||||||
private Map<String, DatabaseReader> databaseReaders;
|
private Map<String, DatabaseReader> databaseReaders;
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public synchronized Map<String, Processor.Factory> getProcessors(
|
public Map<String, Processor.Factory> getProcessors(
|
||||||
Environment env, ScriptService scriptService, TemplateService templateService) {
|
Environment env, ScriptService scriptService, TemplateService templateService) {
|
||||||
if (databaseReaders != null) {
|
if (databaseReaders != null) {
|
||||||
throw new IllegalStateException("called onModule twice for geoip plugin!!");
|
throw new IllegalStateException("called onModule twice for geoip plugin!!");
|
||||||
|
|
|
@ -20,8 +20,14 @@
|
||||||
package org.elasticsearch.ingest.useragent;
|
package org.elasticsearch.ingest.useragent;
|
||||||
|
|
||||||
import org.elasticsearch.common.settings.Setting;
|
import org.elasticsearch.common.settings.Setting;
|
||||||
|
import org.elasticsearch.common.settings.Settings;
|
||||||
|
import org.elasticsearch.env.Environment;
|
||||||
|
import org.elasticsearch.ingest.Processor;
|
||||||
|
import org.elasticsearch.ingest.TemplateService;
|
||||||
import org.elasticsearch.node.NodeModule;
|
import org.elasticsearch.node.NodeModule;
|
||||||
|
import org.elasticsearch.plugins.IngestPlugin;
|
||||||
import org.elasticsearch.plugins.Plugin;
|
import org.elasticsearch.plugins.Plugin;
|
||||||
|
import org.elasticsearch.script.ScriptService;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.io.InputStream;
|
import java.io.InputStream;
|
||||||
|
@ -34,28 +40,36 @@ import java.util.HashMap;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.stream.Stream;
|
import java.util.stream.Stream;
|
||||||
|
|
||||||
public class IngestUserAgentPlugin extends Plugin {
|
public class IngestUserAgentPlugin extends Plugin implements IngestPlugin {
|
||||||
|
|
||||||
private final Setting<Long> CACHE_SIZE_SETTING = Setting.longSetting("ingest.useragent.cache_size", 1000, 0,
|
private final Setting<Long> CACHE_SIZE_SETTING = Setting.longSetting("ingest.useragent.cache_size", 1000, 0,
|
||||||
Setting.Property.NodeScope);
|
Setting.Property.NodeScope);
|
||||||
|
|
||||||
static final String DEFAULT_PARSER_NAME = "_default_";
|
static final String DEFAULT_PARSER_NAME = "_default_";
|
||||||
|
private final Settings settings;
|
||||||
|
|
||||||
public void onModule(NodeModule nodeModule) throws IOException {
|
IngestUserAgentPlugin(Settings settings) {
|
||||||
Path userAgentConfigDirectory = nodeModule.getNode().getEnvironment().configFile().resolve("ingest-useragent");
|
this.settings = settings;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Map<String, Processor.Factory> getProcessors(
|
||||||
|
Environment env, ScriptService scriptService, TemplateService templateService) {
|
||||||
|
Path userAgentConfigDirectory = env.configFile().resolve("ingest-useragent");
|
||||||
|
|
||||||
if (Files.exists(userAgentConfigDirectory) == false && Files.isDirectory(userAgentConfigDirectory)) {
|
if (Files.exists(userAgentConfigDirectory) == false && Files.isDirectory(userAgentConfigDirectory)) {
|
||||||
throw new IllegalStateException(
|
throw new IllegalStateException(
|
||||||
"the user agent directory [" + userAgentConfigDirectory + "] containing the regex file doesn't exist");
|
"the user agent directory [" + userAgentConfigDirectory + "] containing the regex file doesn't exist");
|
||||||
}
|
}
|
||||||
|
|
||||||
long cacheSize = CACHE_SIZE_SETTING.get(nodeModule.getNode().settings());
|
long cacheSize = CACHE_SIZE_SETTING.get(settings);
|
||||||
|
Map<String, UserAgentParser> userAgentParsers;
|
||||||
UserAgentCache cache = new UserAgentCache(cacheSize);
|
try {
|
||||||
|
userAgentParsers = createUserAgentParsers(userAgentConfigDirectory, new UserAgentCache(cacheSize));
|
||||||
Map<String, UserAgentParser> userAgentParsers = createUserAgentParsers(userAgentConfigDirectory, cache);
|
} catch (IOException e) {
|
||||||
|
throw new RuntimeException(e);
|
||||||
nodeModule.registerProcessor(UserAgentProcessor.TYPE, (registry) -> new UserAgentProcessor.Factory(userAgentParsers));
|
}
|
||||||
|
return Collections.singletonMap(UserAgentProcessor.TYPE, new UserAgentProcessor.Factory(userAgentParsers));
|
||||||
}
|
}
|
||||||
|
|
||||||
static Map<String, UserAgentParser> createUserAgentParsers(Path userAgentConfigDirectory, UserAgentCache cache) throws IOException {
|
static Map<String, UserAgentParser> createUserAgentParsers(Path userAgentConfigDirectory, UserAgentCache cache) throws IOException {
|
||||||
|
|
|
@ -194,7 +194,8 @@ public class UserAgentProcessor extends AbstractProcessor {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public UserAgentProcessor create(String processorTag, Map<String, Object> config) throws Exception {
|
public UserAgentProcessor create(Map<String, Processor.Factory> factories, String processorTag,
|
||||||
|
Map<String, Object> config) throws Exception {
|
||||||
String field = readStringProperty(TYPE, processorTag, config, "field");
|
String field = readStringProperty(TYPE, processorTag, config, "field");
|
||||||
String targetField = readStringProperty(TYPE, processorTag, config, "target_field", "useragent");
|
String targetField = readStringProperty(TYPE, processorTag, config, "target_field", "useragent");
|
||||||
String regexFilename = readStringProperty(TYPE, processorTag, config, "regex_file", IngestUserAgentPlugin.DEFAULT_PARSER_NAME);
|
String regexFilename = readStringProperty(TYPE, processorTag, config, "regex_file", IngestUserAgentPlugin.DEFAULT_PARSER_NAME);
|
||||||
|
|
|
@ -81,7 +81,7 @@ public class UserAgentProcessorFactoryTests extends ESTestCase {
|
||||||
|
|
||||||
String processorTag = randomAsciiOfLength(10);
|
String processorTag = randomAsciiOfLength(10);
|
||||||
|
|
||||||
UserAgentProcessor processor = factory.create(processorTag, config);
|
UserAgentProcessor processor = factory.create(null, processorTag, config);
|
||||||
assertThat(processor.getTag(), equalTo(processorTag));
|
assertThat(processor.getTag(), equalTo(processorTag));
|
||||||
assertThat(processor.getField(), equalTo("_field"));
|
assertThat(processor.getField(), equalTo("_field"));
|
||||||
assertThat(processor.getTargetField(), equalTo("useragent"));
|
assertThat(processor.getTargetField(), equalTo("useragent"));
|
||||||
|
@ -98,7 +98,7 @@ public class UserAgentProcessorFactoryTests extends ESTestCase {
|
||||||
config.put("field", "_field");
|
config.put("field", "_field");
|
||||||
config.put("target_field", "_target_field");
|
config.put("target_field", "_target_field");
|
||||||
|
|
||||||
UserAgentProcessor processor = factory.create(null, config);
|
UserAgentProcessor processor = factory.create(null, null, config);
|
||||||
assertThat(processor.getField(), equalTo("_field"));
|
assertThat(processor.getField(), equalTo("_field"));
|
||||||
assertThat(processor.getTargetField(), equalTo("_target_field"));
|
assertThat(processor.getTargetField(), equalTo("_target_field"));
|
||||||
}
|
}
|
||||||
|
@ -110,7 +110,7 @@ public class UserAgentProcessorFactoryTests extends ESTestCase {
|
||||||
config.put("field", "_field");
|
config.put("field", "_field");
|
||||||
config.put("regex_file", regexWithoutDevicesFilename);
|
config.put("regex_file", regexWithoutDevicesFilename);
|
||||||
|
|
||||||
UserAgentProcessor processor = factory.create(null, config);
|
UserAgentProcessor processor = factory.create(null, null, config);
|
||||||
assertThat(processor.getField(), equalTo("_field"));
|
assertThat(processor.getField(), equalTo("_field"));
|
||||||
assertThat(processor.getUaParser().getUaPatterns().size(), greaterThan(0));
|
assertThat(processor.getUaParser().getUaPatterns().size(), greaterThan(0));
|
||||||
assertThat(processor.getUaParser().getOsPatterns().size(), greaterThan(0));
|
assertThat(processor.getUaParser().getOsPatterns().size(), greaterThan(0));
|
||||||
|
@ -124,7 +124,7 @@ public class UserAgentProcessorFactoryTests extends ESTestCase {
|
||||||
config.put("field", "_field");
|
config.put("field", "_field");
|
||||||
config.put("regex_file", "does-not-exist.yaml");
|
config.put("regex_file", "does-not-exist.yaml");
|
||||||
|
|
||||||
ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, config));
|
ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, config));
|
||||||
assertThat(e.getMessage(), equalTo("[regex_file] regex file [does-not-exist.yaml] doesn't exist (has to exist at node startup)"));
|
assertThat(e.getMessage(), equalTo("[regex_file] regex file [does-not-exist.yaml] doesn't exist (has to exist at node startup)"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -144,7 +144,7 @@ public class UserAgentProcessorFactoryTests extends ESTestCase {
|
||||||
config.put("field", "_field");
|
config.put("field", "_field");
|
||||||
config.put("properties", fieldNames);
|
config.put("properties", fieldNames);
|
||||||
|
|
||||||
UserAgentProcessor processor = factory.create(null, config);
|
UserAgentProcessor processor = factory.create(null, null, config);
|
||||||
assertThat(processor.getField(), equalTo("_field"));
|
assertThat(processor.getField(), equalTo("_field"));
|
||||||
assertThat(processor.getProperties(), equalTo(properties));
|
assertThat(processor.getProperties(), equalTo(properties));
|
||||||
}
|
}
|
||||||
|
@ -156,7 +156,7 @@ public class UserAgentProcessorFactoryTests extends ESTestCase {
|
||||||
config.put("field", "_field");
|
config.put("field", "_field");
|
||||||
config.put("properties", Collections.singletonList("invalid"));
|
config.put("properties", Collections.singletonList("invalid"));
|
||||||
|
|
||||||
ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, config));
|
ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, config));
|
||||||
assertThat(e.getMessage(), equalTo("[properties] illegal property value [invalid]. valid values are [NAME, MAJOR, MINOR, "
|
assertThat(e.getMessage(), equalTo("[properties] illegal property value [invalid]. valid values are [NAME, MAJOR, MINOR, "
|
||||||
+ "PATCH, OS, OS_NAME, OS_MAJOR, OS_MINOR, DEVICE, BUILD]"));
|
+ "PATCH, OS, OS_NAME, OS_MAJOR, OS_MINOR, DEVICE, BUILD]"));
|
||||||
}
|
}
|
||||||
|
@ -168,7 +168,7 @@ public class UserAgentProcessorFactoryTests extends ESTestCase {
|
||||||
config.put("field", "_field");
|
config.put("field", "_field");
|
||||||
config.put("properties", "invalid");
|
config.put("properties", "invalid");
|
||||||
|
|
||||||
ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, config));
|
ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, config));
|
||||||
assertThat(e.getMessage(), equalTo("[properties] property isn't a list, but of type [java.lang.String]"));
|
assertThat(e.getMessage(), equalTo("[properties] property isn't a list, but of type [java.lang.String]"));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue