honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add ECS schema for user-agent ingest processor (#37727) (#37984) 

* Add ECS schema for user-agent ingest processor (#37727)

This switches the format of the user agent processor to use the schema from [ECS](https://github.com/elastic/ecs).
So rather than something like this:

```
{
  "patch" : "3538",
  "major" : "70",
  "minor" : "0",
  "os" : "Mac OS X 10.14.1",
  "os_minor" : "14",
  "os_major" : "10",
  "name" : "Chrome",
  "os_name" : "Mac OS X",
  "device" : "Other"
}
```

The structure is now like this:

```
{
  "name" : "Chrome",
  "original" : "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36",
  "os" : {
    "name" : "Mac OS X",
    "version" : "10.14.1",
    "full" : "Mac OS X 10.14.1"
  },
  "device" : "Other",
  "version" : "70.0.3538.102"
}
```

This is now the default for 7.0. The deprecated `ecs` setting in 6.x is not
supported.

Resolves #37329

* Remove `ecs` setting from docs
This commit is contained in:
Lee Hinman 2019-01-30 11:24:18 -07:00 committed by GitHub
parent a070b8acc0
commit cac6b8e06f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 92 additions and 107 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
docs/reference/ingest/processors/user-agent.asciidoc
View File

@ -60,13 +60,13 @@ Which returns
"agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36",
"user_agent": {
"name": "Chrome",
"major": "51",
"minor": "0",
"patch": "2704",
"os_name": "Mac OS X",
"os": "Mac OS X 10.10.5",
"os_major": "10",
"os_minor": "10",
"original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36",
"version": "51.0.2704",
"os": {
"name": "Mac OS X",
"version": "10.10.5",
"full": "Mac OS X 10.10.5"
},
"device": "Other"
}
}

6
docs/reference/migration/migrate_7_0/settings.asciidoc
View File

@ -182,3 +182,9 @@ could have lead to dropping audit events while the operations on the system
were allowed to continue as usual. The recommended replacement is the
use of the `logfile` audit output type and using other components from the
Elastic Stack to handle the indexing part.
[float]
[[ingest-user-agent-ecs-always]]
==== Ingest User Agent processor always uses `ecs` output format
The deprecated `ecs` setting for the user agent ingest processor has been
removed. https://github.com/elastic/ecs[ECS] format is now the default.

87
modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java
View File

@ -19,6 +19,8 @@
package org.elasticsearch.ingest.useragent;
import org.apache.logging.log4j.LogManager;
import org.elasticsearch.common.logging.DeprecationLogger;
import org.elasticsearch.ingest.AbstractProcessor;
import org.elasticsearch.ingest.IngestDocument;
import org.elasticsearch.ingest.Processor;
@ -40,6 +42,8 @@ import static org.elasticsearch.ingest.ConfigurationUtils.readStringProperty;
public class UserAgentProcessor extends AbstractProcessor {
private static final DeprecationLogger deprecationLogger = new DeprecationLogger(LogManager.getLogger(UserAgentProcessor.class));
public static final String TYPE = "user_agent";
private final String field;
@ -63,7 +67,7 @@ public class UserAgentProcessor extends AbstractProcessor {
}
@Override
public IngestDocument execute(IngestDocument ingestDocument) throws Exception {
public IngestDocument execute(IngestDocument ingestDocument) {
String userAgent = ingestDocument.getFieldValue(field, String.class, ignoreMissing);
if (userAgent == null && ignoreMissing) {
@ -75,68 +79,64 @@ public class UserAgentProcessor extends AbstractProcessor {
Details uaClient = parser.parse(userAgent);
Map<String, Object> uaDetails = new HashMap<>();
// Parse the user agent in the ECS (Elastic Common Schema) format
for (Property property : this.properties) {
switch (property) {
case ORIGINAL:
uaDetails.put("original", userAgent);
break;
case NAME:
if (uaClient.userAgent != null && uaClient.userAgent.name != null) {
uaDetails.put("name", uaClient.userAgent.name);
}
else {
} else {
uaDetails.put("name", "Other");
}
break;
case MAJOR:
case VERSION:
StringBuilder version = new StringBuilder();
if (uaClient.userAgent != null && uaClient.userAgent.major != null) {
uaDetails.put("major", uaClient.userAgent.major);
version.append(uaClient.userAgent.major);
if (uaClient.userAgent.minor != null) {
version.append(".").append(uaClient.userAgent.minor);
if (uaClient.userAgent.patch != null) {
version.append(".").append(uaClient.userAgent.patch);
if (uaClient.userAgent.build != null) {
version.append(".").append(uaClient.userAgent.build);
}
break;
case MINOR:
if (uaClient.userAgent != null && uaClient.userAgent.minor != null) {
uaDetails.put("minor", uaClient.userAgent.minor);
}
break;
case PATCH:
if (uaClient.userAgent != null && uaClient.userAgent.patch != null) {
uaDetails.put("patch", uaClient.userAgent.patch);
}
break;
case BUILD:
if (uaClient.userAgent != null && uaClient.userAgent.build != null) {
uaDetails.put("build", uaClient.userAgent.build);
uaDetails.put("version", version.toString());
}
break;
case OS:
if (uaClient.operatingSystem != null) {
uaDetails.put("os", buildFullOSName(uaClient.operatingSystem));
Map<String, String> osDetails = new HashMap<>(3);
if (uaClient.operatingSystem.name != null) {
osDetails.put("name", uaClient.operatingSystem.name);
StringBuilder sb = new StringBuilder();
if (uaClient.operatingSystem.major != null) {
sb.append(uaClient.operatingSystem.major);
if (uaClient.operatingSystem.minor != null) {
sb.append(".").append(uaClient.operatingSystem.minor);
if (uaClient.operatingSystem.patch != null) {
sb.append(".").append(uaClient.operatingSystem.patch);
if (uaClient.operatingSystem.build != null) {
sb.append(".").append(uaClient.operatingSystem.build);
}
else {
uaDetails.put("os", "Other");
}
break;
case OS_NAME:
if (uaClient.operatingSystem != null && uaClient.operatingSystem.name != null) {
uaDetails.put("os_name", uaClient.operatingSystem.name);
}
else {
uaDetails.put("os_name", "Other");
osDetails.put("version", sb.toString());
osDetails.put("full", uaClient.operatingSystem.name + " " + sb.toString());
}
break;
case OS_MAJOR:
if (uaClient.operatingSystem != null && uaClient.operatingSystem.major != null) {
uaDetails.put("os_major", uaClient.operatingSystem.major);
uaDetails.put("os", osDetails);
}
break;
case OS_MINOR:
if (uaClient.operatingSystem != null && uaClient.operatingSystem.minor != null) {
uaDetails.put("os_minor", uaClient.operatingSystem.minor);
}
break;
case DEVICE:
if (uaClient.device != null && uaClient.device.name != null) {
uaDetails.put("device", uaClient.device.name);
}
else {
} else {
uaDetails.put("device", "Other");
}
break;
@ -215,6 +215,10 @@ public class UserAgentProcessor extends AbstractProcessor {
String regexFilename = readStringProperty(TYPE, processorTag, config, "regex_file", IngestUserAgentPlugin.DEFAULT_PARSER_NAME);
List<String> propertyNames = readOptionalList(TYPE, processorTag, config, "properties");
boolean ignoreMissing = readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false);
Object ecsValue = config.remove("ecs");
if (ecsValue != null) {
deprecationLogger.deprecated("setting [ecs] is deprecated as ECS format is the default and only option");
}
UserAgentParser parser = userAgentParsers.get(regexFilename);
if (parser == null) {
@ -242,13 +246,16 @@ public class UserAgentProcessor extends AbstractProcessor {
enum Property {
NAME, MAJOR, MINOR, PATCH, OS, OS_NAME, OS_MAJOR, OS_MINOR, DEVICE, BUILD;
NAME,
OS,
DEVICE,
ORIGINAL,
VERSION;
public static Property parseProperty(String propertyName) {
try {
return valueOf(propertyName.toUpperCase(Locale.ROOT));
}
catch (IllegalArgumentException e) {
} catch (IllegalArgumentException e) {
throw new IllegalArgumentException("illegal property value [" + propertyName + "]. valid values are " +
Arrays.toString(EnumSet.allOf(Property.class).toArray()));
}

4
modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorFactoryTests.java
View File

@ -178,8 +178,8 @@ public class UserAgentProcessorFactoryTests extends ESTestCase {
config.put("properties", Collections.singletonList("invalid"));
ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, config));
assertThat(e.getMessage(), equalTo("[properties] illegal property value [invalid]. valid values are [NAME, MAJOR, MINOR, "
+ "PATCH, OS, OS_NAME, OS_MAJOR, OS_MINOR, DEVICE, BUILD]"));
assertThat(e.getMessage(), equalTo("[properties] illegal property value [invalid]. valid values are [NAME, OS, DEVICE, " +
"ORIGINAL, VERSION]"));
}
public void testInvalidPropertiesType() throws Exception {

44
modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java
View File

@ -103,16 +103,13 @@ public class UserAgentProcessorTests extends ESTestCase {
Map<String, Object> target = (Map<String, Object>) data.get("target_field");
assertThat(target.get("name"), is("Chrome"));
assertThat(target.get("major"), is("33"));
assertThat(target.get("minor"), is("0"));
assertThat(target.get("patch"), is("1750"));
assertNull(target.get("build"));
assertThat(target.get("os"), is("Mac OS X 10.9.2"));
assertThat(target.get("os_name"), is("Mac OS X"));
assertThat(target.get("os_major"), is("10"));
assertThat(target.get("os_minor"), is("9"));
assertThat(target.get("version"), is("33.0.1750"));
Map<String, String> os = new HashMap<>();
os.put("name", "Mac OS X");
os.put("version", "10.9.2");
os.put("full", "Mac OS X 10.9.2");
assertThat(target.get("os"), is(os));
assertThat(target.get("device"), is("Other"));
}
@ -131,15 +128,13 @@ public class UserAgentProcessorTests extends ESTestCase {
Map<String, Object> target = (Map<String, Object>) data.get("target_field");
assertThat(target.get("name"), is("Android"));
assertThat(target.get("major"), is("3"));
assertThat(target.get("minor"), is("0"));
assertNull(target.get("patch"));
assertNull(target.get("build"));
assertThat(target.get("version"), is("3.0"));
assertThat(target.get("os"), is("Android 3.0"));
assertThat(target.get("os_name"), is("Android"));
assertThat(target.get("os_major"), is("3"));
assertThat(target.get("os_minor"), is("0"));
Map<String, String> os = new HashMap<>();
os.put("name", "Android");
os.put("version", "3.0");
os.put("full", "Android 3.0");
assertThat(target.get("os"), is(os));
assertThat(target.get("device"), is("Motorola Xoom"));
}
@ -158,15 +153,9 @@ public class UserAgentProcessorTests extends ESTestCase {
Map<String, Object> target = (Map<String, Object>) data.get("target_field");
assertThat(target.get("name"), is("EasouSpider"));
assertNull(target.get("major"));
assertNull(target.get("minor"));
assertNull(target.get("patch"));
assertNull(target.get("build"));
assertThat(target.get("os"), is("Other"));
assertThat(target.get("os_name"), is("Other"));
assertNull(target.get("os_major"));
assertNull(target.get("os_minor"));
assertNull(target.get("version"));
assertNull(target.get("os"));
assertThat(target.get("device"), is("Spider"));
}
@ -190,10 +179,7 @@ public class UserAgentProcessorTests extends ESTestCase {
assertNull(target.get("patch"));
assertNull(target.get("build"));
assertThat(target.get("os"), is("Other"));
assertThat(target.get("os_name"), is("Other"));
assertNull(target.get("os_major"));
assertNull(target.get("os_minor"));
assertNull(target.get("os"));
assertThat(target.get("device"), is("Other"));
}

19
modules/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/20_useragent_processor.yml
View File

@ -29,13 +29,9 @@
id: 1
- match: { _source.field1: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36" }
- match: { _source.user_agent.name: "Chrome" }
- match: { _source.user_agent.os: "Mac OS X 10.9.2" }
- match: { _source.user_agent.os_name: "Mac OS X" }
- match: { _source.user_agent.os_major: "10" }
- match: { _source.user_agent.os_minor: "9" }
- match: { _source.user_agent.major: "33" }
- match: { _source.user_agent.minor: "0" }
- match: { _source.user_agent.patch: "1750" }
- match: { _source.user_agent.original: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36" }
- match: { _source.user_agent.os: {"name":"Mac OS X", "version":"10.9.2", "full":"Mac OS X 10.9.2"} }
- match: { _source.user_agent.version: "33.0.1750" }
- match: { _source.user_agent.device: "Other" }
---
@ -70,13 +66,8 @@
index: test
id: 1
- match: { _source.field1: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36" }
- match: { _source.field2.os: "Mac OS X 10.9.2" }
- match: { _source.field2.os.full: "Mac OS X 10.9.2" }
- is_false: _source.user_agent
- is_false: _source.field2.name
- is_false: _source.field2.os_name
- is_false: _source.field2.os_major
- is_false: _source.field2.os_minor
- is_false: _source.field2.major
- is_false: _source.field2.minor
- is_false: _source.field2.patch
- is_false: _source.field2.device
- is_false: _source.field2.original

9
modules/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/30_custom_regex.yml
View File

@ -30,11 +30,6 @@
id: 1
- match: { _source.field1: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36" }
- match: { _source.user_agent.name: "Test" }
- match: { _source.user_agent.os: "Other" }
- match: { _source.user_agent.os_name: "Other" }
- match: { _source.user_agent.device: "Other" }
- is_false: _source.user_agent.os_major
- is_false: _source.user_agent.os_minor
- is_false: _source.user_agent.major
- is_false: _source.user_agent.minor
- is_false: _source.user_agent.patch
- is_false: _source.user_agent.os
- is_false: _source.user_agent.version