From 19393fc5a7dd4e7c8082971d6a1b0760ce128b5f Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Thu, 10 Oct 2019 08:45:43 +0200 Subject: [PATCH] match processor should handler values other than string properly (#47419) Currently if the document being ingested contains another field value than a string then the processor fails with an error. This commit changes the match processor to handle number values and array values correctly. If a json array is detected then the `terms` query is used instead of the `term` query. --- .../xpack/enrich/MatchProcessor.java | 8 ++- .../xpack/enrich/MatchProcessorTests.java | 71 ++++++++++++++++++- 2 files changed, 75 insertions(+), 4 deletions(-) diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/MatchProcessor.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/MatchProcessor.java index 55a3b2275e9..104a0f0b267 100644 --- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/MatchProcessor.java +++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/MatchProcessor.java @@ -10,7 +10,9 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.Client; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.TermQueryBuilder; +import org.elasticsearch.index.query.TermsQueryBuilder; +import java.util.List; import java.util.function.BiConsumer; public class MatchProcessor extends AbstractEnrichProcessor { @@ -42,6 +44,10 @@ public class MatchProcessor extends AbstractEnrichProcessor { @Override public QueryBuilder getQueryBuilder(Object fieldValue) { - return new TermQueryBuilder(matchField, fieldValue); + if (fieldValue instanceof List) { + return new TermsQueryBuilder(matchField, (List) fieldValue); + } else { + return new TermQueryBuilder(matchField, fieldValue); + } } } diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/MatchProcessorTests.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/MatchProcessorTests.java index 5dd24e44b8f..26caf72ed20 100644 --- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/MatchProcessorTests.java +++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/MatchProcessorTests.java @@ -21,6 +21,7 @@ import org.elasticsearch.index.VersionType; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.query.ConstantScoreQueryBuilder; import org.elasticsearch.index.query.TermQueryBuilder; +import org.elasticsearch.index.query.TermsQueryBuilder; import org.elasticsearch.ingest.IngestDocument; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; @@ -31,6 +32,7 @@ import org.elasticsearch.test.ESTestCase; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.UncheckedIOException; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -206,6 +208,69 @@ public class MatchProcessorTests extends ESTestCase { assertThat(resultHolder[0].getFieldValue("tld", Object.class), equalTo(null)); } + public void testNumericValue() { + int maxMatches = randomIntBetween(1, 8); + MockSearchFunction mockSearch = mockedSearchFunction(mapOf(2, mapOf("globalRank", 451, "tldRank", 23, "tld", "co"))); + MatchProcessor processor = + new MatchProcessor("_tag", mockSearch, "_name", "domain", "entry", false, true, "domain", maxMatches); + IngestDocument ingestDocument = + new IngestDocument("_index", "_type", "_id", "_routing", 1L, VersionType.INTERNAL, mapOf("domain", 2)); + + // Execute + IngestDocument[] holder = new IngestDocument[1]; + processor.execute(ingestDocument, (result, e) -> holder[0] = result); + assertThat(holder[0], notNullValue()); + + // Check request + SearchRequest request = mockSearch.getCapturedRequest(); + assertThat(request.source().query(), instanceOf(ConstantScoreQueryBuilder.class)); + assertThat(((ConstantScoreQueryBuilder) request.source().query()).innerQuery(), instanceOf(TermQueryBuilder.class)); + TermQueryBuilder termQueryBuilder = (TermQueryBuilder) ((ConstantScoreQueryBuilder) request.source().query()).innerQuery(); + assertThat(termQueryBuilder.fieldName(), equalTo("domain")); + assertThat(termQueryBuilder.value(), equalTo(2)); + + // Check result + List entries = ingestDocument.getFieldValue("entry", List.class); + Map entry = (Map) entries.get(0); + assertThat(entry.size(), equalTo(3)); + assertThat(entry.get("globalRank"), equalTo(451)); + assertThat(entry.get("tldRank"), equalTo(23)); + assertThat(entry.get("tld"), equalTo("co")); + } + + public void testArray() { + int maxMatches = randomIntBetween(1, 8); + MockSearchFunction mockSearch = + mockedSearchFunction(mapOf(Arrays.asList("1", "2"), mapOf("globalRank", 451, "tldRank", 23, "tld", "co"))); + MatchProcessor processor = + new MatchProcessor("_tag", mockSearch, "_name", "domain", "entry", false, true, "domain", maxMatches); + IngestDocument ingestDocument = + new IngestDocument("_index", "_type", "_id", "_routing", 1L, VersionType.INTERNAL, mapOf("domain", Arrays.asList("1", "2"))); + + // Execute + IngestDocument[] holder = new IngestDocument[1]; + processor.execute(ingestDocument, (result, e) -> holder[0] = result); + assertThat(holder[0], notNullValue()); + + // Check request + SearchRequest request = mockSearch.getCapturedRequest(); + assertThat(request.source().query(), instanceOf(ConstantScoreQueryBuilder.class)); + assertThat(((ConstantScoreQueryBuilder) request.source().query()).innerQuery(), instanceOf(TermsQueryBuilder.class)); + TermsQueryBuilder termQueryBuilder = (TermsQueryBuilder) ((ConstantScoreQueryBuilder) request.source().query()).innerQuery(); + assertThat(termQueryBuilder.fieldName(), equalTo("domain")); + assertThat(termQueryBuilder.values().size(), equalTo(2)); + assertThat(termQueryBuilder.values().get(0), equalTo("1")); + assertThat(termQueryBuilder.values().get(1), equalTo("2")); + + // Check result + List entries = ingestDocument.getFieldValue("entry", List.class); + Map entry = (Map) entries.get(0); + assertThat(entry.size(), equalTo(3)); + assertThat(entry.get("globalRank"), equalTo(451)); + assertThat(entry.get("tldRank"), equalTo(23)); + assertThat(entry.get("tld"), equalTo("co")); + } + private static final class MockSearchFunction implements BiConsumer> { private final SearchResponse mockResponse; private final SetOnce capturedRequest; @@ -246,13 +311,13 @@ public class MatchProcessorTests extends ESTestCase { return new MockSearchFunction(exception); } - public MockSearchFunction mockedSearchFunction(Map> documents) { + public MockSearchFunction mockedSearchFunction(Map> documents) { return new MockSearchFunction(mockResponse(documents)); } - public SearchResponse mockResponse(Map> documents) { + public SearchResponse mockResponse(Map> documents) { SearchHit[] searchHits = documents.entrySet().stream().map(e -> { - SearchHit searchHit = new SearchHit(randomInt(100), e.getKey(), new Text(MapperService.SINGLE_MAPPING_NAME), + SearchHit searchHit = new SearchHit(randomInt(100), e.getKey().toString(), new Text(MapperService.SINGLE_MAPPING_NAME), Collections.emptyMap()); try (XContentBuilder builder = XContentBuilder.builder(XContentType.SMILE.xContent())) { builder.map(e.getValue());