honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add ECS schema for user-agent ingest processor (#37727) (#37984) 

* Add ECS schema for user-agent ingest processor (#37727)

This switches the format of the user agent processor to use the schema from [ECS](https://github.com/elastic/ecs).
So rather than something like this:

```
{
  "patch" : "3538",
  "major" : "70",
  "minor" : "0",
  "os" : "Mac OS X 10.14.1",
  "os_minor" : "14",
  "os_major" : "10",
  "name" : "Chrome",
  "os_name" : "Mac OS X",
  "device" : "Other"
}
```

The structure is now like this:

```
{
  "name" : "Chrome",
  "original" : "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36",
  "os" : {
    "name" : "Mac OS X",
    "version" : "10.14.1",
    "full" : "Mac OS X 10.14.1"
  },
  "device" : "Other",
  "version" : "70.0.3538.102"
}
```

This is now the default for 7.0. The deprecated `ecs` setting in 6.x is not
supported.

Resolves #37329

* Remove `ecs` setting from docs
This commit is contained in:
Lee Hinman 2019-01-30 11:24:18 -07:00 committed by GitHub
parent a070b8acc0
commit cac6b8e06f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 92 additions and 107 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
docs/reference/ingest/processors/user-agent.asciidoc
View File

@ -60,13 +60,13 @@ Which returns
"agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36", "agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36",
"user_agent": { "user_agent": {
"name": "Chrome", "name": "Chrome",
"major": "51", "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36",
"minor": "0", "version": "51.0.2704",
"patch": "2704", "os": {
"os_name": "Mac OS X", "name": "Mac OS X",
"os": "Mac OS X 10.10.5", "version": "10.10.5",
"os_major": "10", "full": "Mac OS X 10.10.5"
"os_minor": "10", },
"device": "Other" "device": "Other"
} }
} }

6
docs/reference/migration/migrate_7_0/settings.asciidoc
View File

@ -182,3 +182,9 @@ could have lead to dropping audit events while the operations on the system
were allowed to continue as usual. The recommended replacement is the were allowed to continue as usual. The recommended replacement is the
use of the `logfile` audit output type and using other components from the use of the `logfile` audit output type and using other components from the
Elastic Stack to handle the indexing part. Elastic Stack to handle the indexing part.
[float]
[[ingest-user-agent-ecs-always]]
==== Ingest User Agent processor always uses `ecs` output format
The deprecated `ecs` setting for the user agent ingest processor has been
removed. https://github.com/elastic/ecs[ECS] format is now the default.

87
modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java
View File

@ -19,6 +19,8 @@
package org.elasticsearch.ingest.useragent; package org.elasticsearch.ingest.useragent;
import org.apache.logging.log4j.LogManager;
import org.elasticsearch.common.logging.DeprecationLogger;
import org.elasticsearch.ingest.AbstractProcessor; import org.elasticsearch.ingest.AbstractProcessor;
import org.elasticsearch.ingest.IngestDocument; import org.elasticsearch.ingest.IngestDocument;
import org.elasticsearch.ingest.Processor; import org.elasticsearch.ingest.Processor;
@ -40,6 +42,8 @@ import static org.elasticsearch.ingest.ConfigurationUtils.readStringProperty;
public class UserAgentProcessor extends AbstractProcessor { public class UserAgentProcessor extends AbstractProcessor {
private static final DeprecationLogger deprecationLogger = new DeprecationLogger(LogManager.getLogger(UserAgentProcessor.class));
public static final String TYPE = "user_agent"; public static final String TYPE = "user_agent";
private final String field; private final String field;
@ -63,7 +67,7 @@ public class UserAgentProcessor extends AbstractProcessor {
} }
@Override @Override
public IngestDocument execute(IngestDocument ingestDocument) throws Exception { public IngestDocument execute(IngestDocument ingestDocument) {
String userAgent = ingestDocument.getFieldValue(field, String.class, ignoreMissing); String userAgent = ingestDocument.getFieldValue(field, String.class, ignoreMissing);
if (userAgent == null && ignoreMissing) { if (userAgent == null && ignoreMissing) {
@ -75,68 +79,64 @@ public class UserAgentProcessor extends AbstractProcessor {
Details uaClient = parser.parse(userAgent); Details uaClient = parser.parse(userAgent);
Map<String, Object> uaDetails = new HashMap<>(); Map<String, Object> uaDetails = new HashMap<>();
// Parse the user agent in the ECS (Elastic Common Schema) format
for (Property property : this.properties) { for (Property property : this.properties) {
switch (property) { switch (property) {
case ORIGINAL:
uaDetails.put("original", userAgent);
break;
case NAME: case NAME:
if (uaClient.userAgent != null && uaClient.userAgent.name != null) { if (uaClient.userAgent != null && uaClient.userAgent.name != null) {
uaDetails.put("name", uaClient.userAgent.name); uaDetails.put("name", uaClient.userAgent.name);
} } else {
else {
uaDetails.put("name", "Other"); uaDetails.put("name", "Other");
} }
break; break;
case MAJOR: case VERSION:
StringBuilder version = new StringBuilder();
if (uaClient.userAgent != null && uaClient.userAgent.major != null) { if (uaClient.userAgent != null && uaClient.userAgent.major != null) {
uaDetails.put("major", uaClient.userAgent.major); version.append(uaClient.userAgent.major);
if (uaClient.userAgent.minor != null) {
version.append(".").append(uaClient.userAgent.minor);
if (uaClient.userAgent.patch != null) {
version.append(".").append(uaClient.userAgent.patch);
if (uaClient.userAgent.build != null) {
version.append(".").append(uaClient.userAgent.build);
} }
break;
case MINOR:
if (uaClient.userAgent != null && uaClient.userAgent.minor != null) {
uaDetails.put("minor", uaClient.userAgent.minor);
} }
break;
case PATCH:
if (uaClient.userAgent != null && uaClient.userAgent.patch != null) {
uaDetails.put("patch", uaClient.userAgent.patch);
} }
break; uaDetails.put("version", version.toString());
case BUILD:
if (uaClient.userAgent != null && uaClient.userAgent.build != null) {
uaDetails.put("build", uaClient.userAgent.build);
} }
break; break;
case OS: case OS:
if (uaClient.operatingSystem != null) { if (uaClient.operatingSystem != null) {
uaDetails.put("os", buildFullOSName(uaClient.operatingSystem)); Map<String, String> osDetails = new HashMap<>(3);
if (uaClient.operatingSystem.name != null) {
osDetails.put("name", uaClient.operatingSystem.name);
StringBuilder sb = new StringBuilder();
if (uaClient.operatingSystem.major != null) {
sb.append(uaClient.operatingSystem.major);
if (uaClient.operatingSystem.minor != null) {
sb.append(".").append(uaClient.operatingSystem.minor);
if (uaClient.operatingSystem.patch != null) {
sb.append(".").append(uaClient.operatingSystem.patch);
if (uaClient.operatingSystem.build != null) {
sb.append(".").append(uaClient.operatingSystem.build);
} }
else {
uaDetails.put("os", "Other");
} }
break;
case OS_NAME:
if (uaClient.operatingSystem != null && uaClient.operatingSystem.name != null) {
uaDetails.put("os_name", uaClient.operatingSystem.name);
} }
else { osDetails.put("version", sb.toString());
uaDetails.put("os_name", "Other"); osDetails.put("full", uaClient.operatingSystem.name + " " + sb.toString());
} }
break; uaDetails.put("os", osDetails);
case OS_MAJOR:
if (uaClient.operatingSystem != null && uaClient.operatingSystem.major != null) {
uaDetails.put("os_major", uaClient.operatingSystem.major);
} }
break;
case OS_MINOR:
if (uaClient.operatingSystem != null && uaClient.operatingSystem.minor != null) {
uaDetails.put("os_minor", uaClient.operatingSystem.minor);
} }
break; break;
case DEVICE: case DEVICE:
if (uaClient.device != null && uaClient.device.name != null) { if (uaClient.device != null && uaClient.device.name != null) {
uaDetails.put("device", uaClient.device.name); uaDetails.put("device", uaClient.device.name);
} } else {
else {
uaDetails.put("device", "Other"); uaDetails.put("device", "Other");
} }
break; break;
@ -215,6 +215,10 @@ public class UserAgentProcessor extends AbstractProcessor {
String regexFilename = readStringProperty(TYPE, processorTag, config, "regex_file", IngestUserAgentPlugin.DEFAULT_PARSER_NAME); String regexFilename = readStringProperty(TYPE, processorTag, config, "regex_file", IngestUserAgentPlugin.DEFAULT_PARSER_NAME);
List<String> propertyNames = readOptionalList(TYPE, processorTag, config, "properties"); List<String> propertyNames = readOptionalList(TYPE, processorTag, config, "properties");
boolean ignoreMissing = readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false); boolean ignoreMissing = readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false);
Object ecsValue = config.remove("ecs");
if (ecsValue != null) {
deprecationLogger.deprecated("setting [ecs] is deprecated as ECS format is the default and only option");
}
UserAgentParser parser = userAgentParsers.get(regexFilename); UserAgentParser parser = userAgentParsers.get(regexFilename);
if (parser == null) { if (parser == null) {
@ -242,13 +246,16 @@ public class UserAgentProcessor extends AbstractProcessor {
enum Property { enum Property {
NAME, MAJOR, MINOR, PATCH, OS, OS_NAME, OS_MAJOR, OS_MINOR, DEVICE, BUILD; NAME,
OS,
DEVICE,
ORIGINAL,
VERSION;
public static Property parseProperty(String propertyName) { public static Property parseProperty(String propertyName) {
try { try {
return valueOf(propertyName.toUpperCase(Locale.ROOT)); return valueOf(propertyName.toUpperCase(Locale.ROOT));
} } catch (IllegalArgumentException e) {
catch (IllegalArgumentException e) {
throw new IllegalArgumentException("illegal property value [" + propertyName + "]. valid values are " + throw new IllegalArgumentException("illegal property value [" + propertyName + "]. valid values are " +
Arrays.toString(EnumSet.allOf(Property.class).toArray())); Arrays.toString(EnumSet.allOf(Property.class).toArray()));
} }

4
modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorFactoryTests.java
View File

@ -178,8 +178,8 @@ public class UserAgentProcessorFactoryTests extends ESTestCase {
config.put("properties", Collections.singletonList("invalid")); config.put("properties", Collections.singletonList("invalid"));
ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, config)); ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, config));
assertThat(e.getMessage(), equalTo("[properties] illegal property value [invalid]. valid values are [NAME, MAJOR, MINOR, " assertThat(e.getMessage(), equalTo("[properties] illegal property value [invalid]. valid values are [NAME, OS, DEVICE, " +
+ "PATCH, OS, OS_NAME, OS_MAJOR, OS_MINOR, DEVICE, BUILD]")); "ORIGINAL, VERSION]"));
} }
public void testInvalidPropertiesType() throws Exception { public void testInvalidPropertiesType() throws Exception {

44
modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java
View File

@ -103,16 +103,13 @@ public class UserAgentProcessorTests extends ESTestCase {
Map<String, Object> target = (Map<String, Object>) data.get("target_field"); Map<String, Object> target = (Map<String, Object>) data.get("target_field");
assertThat(target.get("name"), is("Chrome")); assertThat(target.get("name"), is("Chrome"));
assertThat(target.get("major"), is("33")); assertThat(target.get("version"), is("33.0.1750"));
assertThat(target.get("minor"), is("0"));
assertThat(target.get("patch"), is("1750"));
assertNull(target.get("build"));
assertThat(target.get("os"), is("Mac OS X 10.9.2"));
assertThat(target.get("os_name"), is("Mac OS X"));
assertThat(target.get("os_major"), is("10"));
assertThat(target.get("os_minor"), is("9"));
Map<String, String> os = new HashMap<>();
os.put("name", "Mac OS X");
os.put("version", "10.9.2");
os.put("full", "Mac OS X 10.9.2");
assertThat(target.get("os"), is(os));
assertThat(target.get("device"), is("Other")); assertThat(target.get("device"), is("Other"));
} }
@ -131,15 +128,13 @@ public class UserAgentProcessorTests extends ESTestCase {
Map<String, Object> target = (Map<String, Object>) data.get("target_field"); Map<String, Object> target = (Map<String, Object>) data.get("target_field");
assertThat(target.get("name"), is("Android")); assertThat(target.get("name"), is("Android"));
assertThat(target.get("major"), is("3")); assertThat(target.get("version"), is("3.0"));
assertThat(target.get("minor"), is("0"));
assertNull(target.get("patch"));
assertNull(target.get("build"));
assertThat(target.get("os"), is("Android 3.0")); Map<String, String> os = new HashMap<>();
assertThat(target.get("os_name"), is("Android")); os.put("name", "Android");
assertThat(target.get("os_major"), is("3")); os.put("version", "3.0");
assertThat(target.get("os_minor"), is("0")); os.put("full", "Android 3.0");
assertThat(target.get("os"), is(os));
assertThat(target.get("device"), is("Motorola Xoom")); assertThat(target.get("device"), is("Motorola Xoom"));
} }
@ -158,15 +153,9 @@ public class UserAgentProcessorTests extends ESTestCase {
Map<String, Object> target = (Map<String, Object>) data.get("target_field"); Map<String, Object> target = (Map<String, Object>) data.get("target_field");
assertThat(target.get("name"), is("EasouSpider")); assertThat(target.get("name"), is("EasouSpider"));
assertNull(target.get("major"));
assertNull(target.get("minor"));
assertNull(target.get("patch"));
assertNull(target.get("build"));
assertThat(target.get("os"), is("Other")); assertNull(target.get("version"));
assertThat(target.get("os_name"), is("Other")); assertNull(target.get("os"));
assertNull(target.get("os_major"));
assertNull(target.get("os_minor"));
assertThat(target.get("device"), is("Spider")); assertThat(target.get("device"), is("Spider"));
} }
@ -190,10 +179,7 @@ public class UserAgentProcessorTests extends ESTestCase {
assertNull(target.get("patch")); assertNull(target.get("patch"));
assertNull(target.get("build")); assertNull(target.get("build"));
assertThat(target.get("os"), is("Other")); assertNull(target.get("os"));
assertThat(target.get("os_name"), is("Other"));
assertNull(target.get("os_major"));
assertNull(target.get("os_minor"));
assertThat(target.get("device"), is("Other")); assertThat(target.get("device"), is("Other"));
} }

19
modules/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/20_useragent_processor.yml
View File

@ -29,13 +29,9 @@
id: 1 id: 1
- match: { _source.field1: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36" } - match: { _source.field1: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36" }
- match: { _source.user_agent.name: "Chrome" } - match: { _source.user_agent.name: "Chrome" }
- match: { _source.user_agent.os: "Mac OS X 10.9.2" } - match: { _source.user_agent.original: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36" }
- match: { _source.user_agent.os_name: "Mac OS X" } - match: { _source.user_agent.os: {"name":"Mac OS X", "version":"10.9.2", "full":"Mac OS X 10.9.2"} }
- match: { _source.user_agent.os_major: "10" } - match: { _source.user_agent.version: "33.0.1750" }
- match: { _source.user_agent.os_minor: "9" }
- match: { _source.user_agent.major: "33" }
- match: { _source.user_agent.minor: "0" }
- match: { _source.user_agent.patch: "1750" }
- match: { _source.user_agent.device: "Other" } - match: { _source.user_agent.device: "Other" }
--- ---
@ -70,13 +66,8 @@
index: test index: test
id: 1 id: 1
- match: { _source.field1: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36" } - match: { _source.field1: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36" }
- match: { _source.field2.os: "Mac OS X 10.9.2" } - match: { _source.field2.os.full: "Mac OS X 10.9.2" }
- is_false: _source.user_agent - is_false: _source.user_agent
- is_false: _source.field2.name - is_false: _source.field2.name
- is_false: _source.field2.os_name
- is_false: _source.field2.os_major
- is_false: _source.field2.os_minor
- is_false: _source.field2.major
- is_false: _source.field2.minor
- is_false: _source.field2.patch
- is_false: _source.field2.device - is_false: _source.field2.device
- is_false: _source.field2.original

9
modules/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/30_custom_regex.yml
View File

@ -30,11 +30,6 @@
id: 1 id: 1
- match: { _source.field1: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36" } - match: { _source.field1: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36" }
- match: { _source.user_agent.name: "Test" } - match: { _source.user_agent.name: "Test" }
- match: { _source.user_agent.os: "Other" }
- match: { _source.user_agent.os_name: "Other" }
- match: { _source.user_agent.device: "Other" } - match: { _source.user_agent.device: "Other" }
- is_false: _source.user_agent.os_major - is_false: _source.user_agent.os
- is_false: _source.user_agent.os_minor - is_false: _source.user_agent.version
- is_false: _source.user_agent.major
- is_false: _source.user_agent.minor
- is_false: _source.user_agent.patch