From d709b8bb97c7c60139dd2bb37bd605160e588598 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Tue, 7 May 2019 08:38:08 +0200 Subject: [PATCH] Rename enrich policy index_pattern field to indices. (#41836) Relates to #32789 --- .../xpack/core/enrich/EnrichPolicy.java | 28 +++++++++---------- .../elasticsearch/xpack/enrich/EnrichIT.java | 2 +- .../rest-api-spec/test/enrich/10_basic.yml | 4 +-- .../xpack/enrich/EnrichPolicyRunner.java | 10 +++---- .../xpack/enrich/EnrichPolicyRunnerTests.java | 6 ++-- .../xpack/enrich/EnrichPolicyTests.java | 4 +-- .../enrich/EnrichProcessorFactoryTests.java | 12 +++++--- .../enrich/ExactMatchProcessorTests.java | 25 +++++++++-------- 8 files changed, 49 insertions(+), 42 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/EnrichPolicy.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/EnrichPolicy.java index cb104d516fb..f8c036fd2ee 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/EnrichPolicy.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/EnrichPolicy.java @@ -36,7 +36,7 @@ public final class EnrichPolicy implements Writeable, ToXContentFragment { static final ParseField TYPE = new ParseField("type"); static final ParseField QUERY = new ParseField("query"); - static final ParseField INDEX_PATTERN = new ParseField("index_pattern"); + static final ParseField INDICES = new ParseField("indices"); static final ParseField ENRICH_KEY = new ParseField("enrich_key"); static final ParseField ENRICH_VALUES = new ParseField("enrich_values"); static final ParseField SCHEDULE = new ParseField("schedule"); @@ -47,7 +47,7 @@ public final class EnrichPolicy implements Writeable, ToXContentFragment { return new EnrichPolicy( (String) args[0], (QuerySource) args[1], - (String) args[2], + (List) args[2], (String) args[3], (List) args[4], (String) args[5] @@ -66,7 +66,7 @@ public final class EnrichPolicy implements Writeable, ToXContentFragment { contentBuilder.generator().copyCurrentStructure(p); return new QuerySource(BytesReference.bytes(contentBuilder), contentBuilder.contentType()); }, QUERY); - parser.declareString(ConstructingObjectParser.constructorArg(), INDEX_PATTERN); + parser.declareStringArray(ConstructingObjectParser.constructorArg(), INDICES); parser.declareString(ConstructingObjectParser.constructorArg(), ENRICH_KEY); parser.declareStringArray(ConstructingObjectParser.constructorArg(), ENRICH_VALUES); parser.declareString(ConstructingObjectParser.constructorArg(), SCHEDULE); @@ -78,7 +78,7 @@ public final class EnrichPolicy implements Writeable, ToXContentFragment { private final String type; private final QuerySource query; - private final String indexPattern; + private final List indices; private final String enrichKey; private final List enrichValues; private final String schedule; @@ -87,7 +87,7 @@ public final class EnrichPolicy implements Writeable, ToXContentFragment { this( in.readString(), in.readOptionalWriteable(QuerySource::new), - in.readString(), + in.readStringList(), in.readString(), in.readStringList(), in.readString() @@ -96,14 +96,14 @@ public final class EnrichPolicy implements Writeable, ToXContentFragment { public EnrichPolicy(String type, QuerySource query, - String indexPattern, + List indices, String enrichKey, List enrichValues, String schedule) { this.type = type; this.query= query; this.schedule = schedule; - this.indexPattern = indexPattern; + this.indices = indices; this.enrichKey = enrichKey; this.enrichValues = enrichValues; } @@ -116,8 +116,8 @@ public final class EnrichPolicy implements Writeable, ToXContentFragment { return query; } - public String getIndexPattern() { - return indexPattern; + public List getIndices() { + return indices; } public String getEnrichKey() { @@ -140,7 +140,7 @@ public final class EnrichPolicy implements Writeable, ToXContentFragment { public void writeTo(StreamOutput out) throws IOException { out.writeString(type); out.writeOptionalWriteable(query); - out.writeString(indexPattern); + out.writeStringCollection(indices); out.writeString(enrichKey); out.writeStringCollection(enrichValues); out.writeString(schedule); @@ -152,7 +152,7 @@ public final class EnrichPolicy implements Writeable, ToXContentFragment { if (query != null) { builder.field(QUERY.getPreferredName(), query.getQueryAsMap()); } - builder.field(INDEX_PATTERN.getPreferredName(), indexPattern); + builder.array(INDICES.getPreferredName(), indices.toArray(new String[0])); builder.field(ENRICH_KEY.getPreferredName(), enrichKey); builder.array(ENRICH_VALUES.getPreferredName(), enrichValues.toArray(new String[0])); builder.field(SCHEDULE.getPreferredName(), schedule); @@ -166,7 +166,7 @@ public final class EnrichPolicy implements Writeable, ToXContentFragment { EnrichPolicy policy = (EnrichPolicy) o; return type.equals(policy.type) && Objects.equals(query, policy.query) && - indexPattern.equals(policy.indexPattern) && + indices.equals(policy.indices) && enrichKey.equals(policy.enrichKey) && enrichValues.equals(policy.enrichValues) && schedule.equals(policy.schedule); @@ -177,7 +177,7 @@ public final class EnrichPolicy implements Writeable, ToXContentFragment { return Objects.hash( type, query, - indexPattern, + indices, enrichKey, enrichValues, schedule @@ -245,7 +245,7 @@ public final class EnrichPolicy implements Writeable, ToXContentFragment { (String) args[0], new EnrichPolicy((String) args[1], (QuerySource) args[2], - (String) args[3], + (List) args[3], (String) args[4], (List) args[5], (String) args[6]) diff --git a/x-pack/plugin/enrich/qa/rest/src/test/java/org/elasticsearch/xpack/enrich/EnrichIT.java b/x-pack/plugin/enrich/qa/rest/src/test/java/org/elasticsearch/xpack/enrich/EnrichIT.java index 9faf345618d..36519d68f0b 100644 --- a/x-pack/plugin/enrich/qa/rest/src/test/java/org/elasticsearch/xpack/enrich/EnrichIT.java +++ b/x-pack/plugin/enrich/qa/rest/src/test/java/org/elasticsearch/xpack/enrich/EnrichIT.java @@ -41,7 +41,7 @@ public class EnrichIT extends ESRestTestCase { public void testBasicFlow() throws Exception { // Create the policy: Request putPolicyRequest = new Request("PUT", "/_enrich/policy/my_policy"); - putPolicyRequest.setJsonEntity("{\"type\": \"exact_match\",\"index_pattern\": \"my-index*\", \"enrich_key\": \"host\", " + + putPolicyRequest.setJsonEntity("{\"type\": \"exact_match\",\"indices\": [\"my-index*\"], \"enrich_key\": \"host\", " + "\"enrich_values\": [\"globalRank\", \"tldRank\", \"tld\"], \"schedule\": \"0 5 * * *\"}"); assertOK(client().performRequest(putPolicyRequest)); diff --git a/x-pack/plugin/enrich/qa/rest/src/test/resources/rest-api-spec/test/enrich/10_basic.yml b/x-pack/plugin/enrich/qa/rest/src/test/resources/rest-api-spec/test/enrich/10_basic.yml index a51a930b6f3..48036ea05b7 100644 --- a/x-pack/plugin/enrich/qa/rest/src/test/resources/rest-api-spec/test/enrich/10_basic.yml +++ b/x-pack/plugin/enrich/qa/rest/src/test/resources/rest-api-spec/test/enrich/10_basic.yml @@ -6,7 +6,7 @@ name: policy-crud body: type: exact_match - index_pattern: "bar*" + indices: ["bar*"] enrich_key: baz enrich_values: ["a", "b"] schedule: "*/120" @@ -17,7 +17,7 @@ - length: { policies: 1 } - match: { policies.0.name: policy-crud } - match: { policies.0.type: exact_match } - - match: { policies.0.index_pattern: "bar*" } + - match: { policies.0.indices: ["bar*"] } - match: { policies.0.enrich_key: baz } - match: { policies.0.enrich_values: ["a", "b"] } - match: { policies.0.schedule: "*/120" } diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPolicyRunner.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPolicyRunner.java index d0509e73a30..2253cad040b 100644 --- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPolicyRunner.java +++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPolicyRunner.java @@ -73,9 +73,9 @@ public class EnrichPolicyRunner implements Runnable { public void run() { // Collect the source index information logger.info("Policy [{}]: Running enrich policy", policyName); - final String sourceIndexPattern = policy.getIndexPattern(); - logger.debug("Policy [{}]: Checking source index [{}]", policyName, sourceIndexPattern); - GetIndexRequest getIndexRequest = new GetIndexRequest().indices(sourceIndexPattern); + final String[] sourceIndices = policy.getIndices().toArray(new String[0]); + logger.debug("Policy [{}]: Checking source indices [{}]", policyName, sourceIndices); + GetIndexRequest getIndexRequest = new GetIndexRequest().indices(sourceIndices); client.admin().indices().getIndex(getIndexRequest, new ActionListener() { @Override public void onResponse(GetIndexResponse getIndexResponse) { @@ -108,7 +108,7 @@ public class EnrichPolicyRunner implements Runnable { listener.onFailure( new ElasticsearchException( "Enrich policy execution for [{}] failed. Could not read mapping for source [{}] included by pattern [{}]", - policyName, sourceIndex, policy.getIndexPattern())); + policyName, sourceIndex, policy.getIndices())); } if (properties.contains(policy.getEnrichKey()) == false) { listener.onFailure( @@ -187,7 +187,7 @@ public class EnrichPolicyRunner implements Runnable { } ReindexRequest reindexRequest = new ReindexRequest() .setDestIndex(destinationIndexName) - .setSourceIndices(policy.getIndexPattern()); + .setSourceIndices(policy.getIndices().toArray(new String[0])); reindexRequest.getSearchRequest().source(searchSourceBuilder); reindexRequest.getDestination().source(new BytesArray(new byte[0]), XContentType.SMILE); client.execute(ReindexAction.INSTANCE, reindexRequest, new ActionListener() { diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichPolicyRunnerTests.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichPolicyRunnerTests.java index 3d42239910a..f36eb75ea47 100644 --- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichPolicyRunnerTests.java +++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichPolicyRunnerTests.java @@ -87,7 +87,8 @@ public class EnrichPolicyRunnerTests extends ESSingleNodeTestCase { List enrichFields = new ArrayList<>(); enrichFields.add("field2"); enrichFields.add("field5"); - EnrichPolicy policy = new EnrichPolicy(EnrichPolicy.EXACT_MATCH_TYPE, null, sourceIndex, "field1", enrichFields, ""); + EnrichPolicy policy = + new EnrichPolicy(EnrichPolicy.EXACT_MATCH_TYPE, null, Collections.singletonList(sourceIndex), "field1", enrichFields, ""); String policyName = "test1"; ActionListener listener = new ActionListener() { @@ -202,7 +203,8 @@ public class EnrichPolicyRunnerTests extends ESSingleNodeTestCase { enrichFields.add("idx"); enrichFields.add("field2"); enrichFields.add("field5"); - EnrichPolicy policy = new EnrichPolicy(EnrichPolicy.EXACT_MATCH_TYPE, null, sourceIndexPattern, "field1", enrichFields, ""); + EnrichPolicy policy = new EnrichPolicy(EnrichPolicy.EXACT_MATCH_TYPE, null, Collections.singletonList(sourceIndexPattern), + "field1", enrichFields, ""); String policyName = "test1"; ActionListener listener = new ActionListener() { diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichPolicyTests.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichPolicyTests.java index 72b5cec3be0..1e48330e849 100644 --- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichPolicyTests.java +++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichPolicyTests.java @@ -59,7 +59,7 @@ public class EnrichPolicyTests extends AbstractSerializingTestCase return new EnrichPolicy( randomFrom(EnrichPolicy.SUPPORTED_POLICY_TYPES), randomBoolean() ? querySource : null, - randomAlphaOfLength(4), + Arrays.asList(generateRandomStringArray(8, 4, false, false)), randomAlphaOfLength(4), Arrays.asList(generateRandomStringArray(8, 4, false, false)), randomAlphaOfLength(4) @@ -89,7 +89,7 @@ public class EnrichPolicyTests extends AbstractSerializingTestCase } else { assertThat(expectedInstance.getQuery(), nullValue()); } - assertThat(newInstance.getIndexPattern(), equalTo(expectedInstance.getIndexPattern())); + assertThat(newInstance.getIndices(), equalTo(expectedInstance.getIndices())); assertThat(newInstance.getEnrichKey(), equalTo(expectedInstance.getEnrichKey())); assertThat(newInstance.getEnrichValues(), equalTo(expectedInstance.getEnrichValues())); assertThat(newInstance.getSchedule(), equalTo(expectedInstance.getSchedule())); diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichProcessorFactoryTests.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichProcessorFactoryTests.java index 249c743e053..1d6228ed602 100644 --- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichProcessorFactoryTests.java +++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichProcessorFactoryTests.java @@ -30,7 +30,8 @@ public class EnrichProcessorFactoryTests extends ESTestCase { public void testCreateProcessorInstance() throws Exception { List enrichValues = Arrays.asList("globalRank", "tldRank", "tld"); - EnrichPolicy policy = new EnrichPolicy(EnrichPolicy.EXACT_MATCH_TYPE, null, "source_index", "my_key", enrichValues, "schedule"); + EnrichPolicy policy = new EnrichPolicy(EnrichPolicy.EXACT_MATCH_TYPE, null, Collections.singletonList("source_index"), "my_key", + enrichValues, "schedule"); EnrichProcessorFactory factory = new EnrichProcessorFactory(createClusterStateSupplier("majestic", policy), null); Map config = new HashMap<>(); @@ -103,7 +104,8 @@ public class EnrichProcessorFactoryTests extends ESTestCase { public void testPolicyNameMissing() { List enrichValues = Arrays.asList("globalRank", "tldRank", "tld"); - EnrichPolicy policy = new EnrichPolicy(EnrichPolicy.EXACT_MATCH_TYPE, null, "source_index", "my_key", enrichValues, "schedule"); + EnrichPolicy policy = new EnrichPolicy(EnrichPolicy.EXACT_MATCH_TYPE, null, Collections.singletonList("source_index"), "my_key", + enrichValues, "schedule"); EnrichProcessorFactory factory = new EnrichProcessorFactory(createClusterStateSupplier("_name", policy), null); Map config = new HashMap<>(); @@ -134,7 +136,8 @@ public class EnrichProcessorFactoryTests extends ESTestCase { public void testUnsupportedPolicy() { List enrichValues = Arrays.asList("globalRank", "tldRank", "tld"); - EnrichPolicy policy = new EnrichPolicy("unsupported", null, "source_index", "my_key", enrichValues, "schedule"); + EnrichPolicy policy = + new EnrichPolicy("unsupported", null, Collections.singletonList("source_index"), "my_key", enrichValues, "schedule"); EnrichProcessorFactory factory = new EnrichProcessorFactory(createClusterStateSupplier("majestic", policy), null); Map config = new HashMap<>(); @@ -166,7 +169,8 @@ public class EnrichProcessorFactoryTests extends ESTestCase { public void testNonExistingDecorateField() throws Exception { List enrichValues = Arrays.asList("globalRank", "tldRank", "tld"); - EnrichPolicy policy = new EnrichPolicy(EnrichPolicy.EXACT_MATCH_TYPE, null, "source_index", "my_key", enrichValues, "schedule"); + EnrichPolicy policy = new EnrichPolicy(EnrichPolicy.EXACT_MATCH_TYPE, null, Collections.singletonList("source_index"), "my_key", + enrichValues, "schedule"); EnrichProcessorFactory factory = new EnrichProcessorFactory(createClusterStateSupplier("majestic", policy), null); Map config = new HashMap<>(); diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/ExactMatchProcessorTests.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/ExactMatchProcessorTests.java index 683bd18b945..265eec7c004 100644 --- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/ExactMatchProcessorTests.java +++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/ExactMatchProcessorTests.java @@ -53,8 +53,8 @@ public class ExactMatchProcessorTests extends ESTestCase { indexWriter.addDocument(createEnrichDocument("eops.nl", "globalRank", 4567, "tldRank", 80, "tld", "nl")); indexWriter.commit(); - EnrichPolicy policy = new EnrichPolicy(EnrichPolicy.EXACT_MATCH_TYPE, null, "majestic_index", "key", - Collections.emptyList(), "schedule"); + EnrichPolicy policy = new EnrichPolicy(EnrichPolicy.EXACT_MATCH_TYPE, null, + Collections.singletonList("majestic_index"), "key", Collections.emptyList(), "schedule"); Function policyLookup = policyName -> policy; try (IndexReader indexReader = DirectoryReader.open(directory)) { @@ -101,8 +101,8 @@ public class ExactMatchProcessorTests extends ESTestCase { indexWriter.addDocument(createEnrichDocument("eops.nl", "globalRank", 4567, "tldRank", 80, "tld", "nl")); indexWriter.commit(); - EnrichPolicy policy = new EnrichPolicy(EnrichPolicy.EXACT_MATCH_TYPE, null, "majestic_index", "key", - Collections.emptyList(), "schedule"); + EnrichPolicy policy = new EnrichPolicy(EnrichPolicy.EXACT_MATCH_TYPE, null, Collections.singletonList("majestic_index"), + "key", Collections.emptyList(), "schedule"); Function policyLookup = policyName -> policy; try (IndexReader indexReader = DirectoryReader.open(directory)) { @@ -133,8 +133,8 @@ public class ExactMatchProcessorTests extends ESTestCase { indexWriter.addDocument(createEnrichDocument("eops.nl", "globalRank", 4567, "tldRank", 80, "tld", "nl")); indexWriter.commit(); - EnrichPolicy policy = new EnrichPolicy(EnrichPolicy.EXACT_MATCH_TYPE, null, "majestic_index", "key", - Collections.emptyList(), "schedule"); + EnrichPolicy policy = new EnrichPolicy(EnrichPolicy.EXACT_MATCH_TYPE, null, Collections.singletonList("majestic_index"), + "key", Collections.emptyList(), "schedule"); Function policyLookup = policyName -> policy; try (IndexReader indexReader = DirectoryReader.open(directory)) { @@ -161,8 +161,8 @@ public class ExactMatchProcessorTests extends ESTestCase { try (IndexWriter indexWriter = new IndexWriter(directory, iwConfig)) { indexWriter.commit(); - EnrichPolicy policy = new EnrichPolicy(EnrichPolicy.EXACT_MATCH_TYPE, null, "majestic_index", "key", - Collections.emptyList(), "schedule"); + EnrichPolicy policy = new EnrichPolicy(EnrichPolicy.EXACT_MATCH_TYPE, null, Collections.singletonList("majestic_index"), + "key", Collections.emptyList(), "schedule"); Function policyLookup = policyName -> policy; try (IndexReader indexReader = DirectoryReader.open(directory)) { @@ -193,8 +193,8 @@ public class ExactMatchProcessorTests extends ESTestCase { indexWriter.addDocument(document); indexWriter.commit(); - EnrichPolicy policy = new EnrichPolicy(EnrichPolicy.EXACT_MATCH_TYPE, null, "majestic_index", "key", - Collections.emptyList(), "schedule"); + EnrichPolicy policy = new EnrichPolicy(EnrichPolicy.EXACT_MATCH_TYPE, null, Collections.singletonList("majestic_index"), + "key", Collections.emptyList(), "schedule"); Function policyLookup = policyName -> policy; try (IndexReader indexReader = DirectoryReader.open(directory)) { @@ -224,8 +224,9 @@ public class ExactMatchProcessorTests extends ESTestCase { } public void testIgnoreKeyMissing() throws Exception { - EnrichPolicy policy = new EnrichPolicy(EnrichPolicy.EXACT_MATCH_TYPE, null, "majestic_index", "key", - Collections.emptyList(), "schedule"); + EnrichPolicy policy = new EnrichPolicy(EnrichPolicy.EXACT_MATCH_TYPE, null, Collections.singletonList("majestic_index"), "key", + Collections.emptyList(), + "schedule"); Function policyLookup = policyName -> policy; { ExactMatchProcessor processor = new ExactMatchProcessor("_tag", policyLookup, indexExpression -> null, "_name", "domain",