honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Rename enrich policy index_pattern field to indices. (#41836) 

Relates to #32789
This commit is contained in:
Martijn van Groningen 2019-05-07 08:38:08 +02:00
parent f366f56f00
commit d709b8bb97
No known key found for this signature in database
GPG Key ID: AB236F4FCF2AF12A
8 changed files with 49 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/EnrichPolicy.java
View File

@ -36,7 +36,7 @@ public final class EnrichPolicy implements Writeable, ToXContentFragment {
static final ParseField TYPE = new ParseField("type");
static final ParseField QUERY = new ParseField("query");
static final ParseField INDEX_PATTERN = new ParseField("index_pattern");
static final ParseField INDICES = new ParseField("indices");
static final ParseField ENRICH_KEY = new ParseField("enrich_key");
static final ParseField ENRICH_VALUES = new ParseField("enrich_values");
static final ParseField SCHEDULE = new ParseField("schedule");
@ -47,7 +47,7 @@ public final class EnrichPolicy implements Writeable, ToXContentFragment {
return new EnrichPolicy(
(String) args[0],
(QuerySource) args[1],
(String) args[2],
(List<String>) args[2],
(String) args[3],
(List<String>) args[4],
(String) args[5]
@ -66,7 +66,7 @@ public final class EnrichPolicy implements Writeable, ToXContentFragment {
contentBuilder.generator().copyCurrentStructure(p);
return new QuerySource(BytesReference.bytes(contentBuilder), contentBuilder.contentType());
}, QUERY);
parser.declareString(ConstructingObjectParser.constructorArg(), INDEX_PATTERN);
parser.declareStringArray(ConstructingObjectParser.constructorArg(), INDICES);
parser.declareString(ConstructingObjectParser.constructorArg(), ENRICH_KEY);
parser.declareStringArray(ConstructingObjectParser.constructorArg(), ENRICH_VALUES);
parser.declareString(ConstructingObjectParser.constructorArg(), SCHEDULE);
@ -78,7 +78,7 @@ public final class EnrichPolicy implements Writeable, ToXContentFragment {
private final String type;
private final QuerySource query;
private final String indexPattern;
private final List<String> indices;
private final String enrichKey;
private final List<String> enrichValues;
private final String schedule;
@ -87,7 +87,7 @@ public final class EnrichPolicy implements Writeable, ToXContentFragment {
this(
in.readString(),
in.readOptionalWriteable(QuerySource::new),
in.readString(),
in.readStringList(),
in.readString(),
in.readStringList(),
in.readString()
@ -96,14 +96,14 @@ public final class EnrichPolicy implements Writeable, ToXContentFragment {
public EnrichPolicy(String type,
QuerySource query,
String indexPattern,
List<String> indices,
String enrichKey,
List<String> enrichValues,
String schedule) {
this.type = type;
this.query= query;
this.schedule = schedule;
this.indexPattern = indexPattern;
this.indices = indices;
this.enrichKey = enrichKey;
this.enrichValues = enrichValues;
}
@ -116,8 +116,8 @@ public final class EnrichPolicy implements Writeable, ToXContentFragment {
return query;
}
public String getIndexPattern() {
return indexPattern;
public List<String> getIndices() {
return indices;
}
public String getEnrichKey() {
@ -140,7 +140,7 @@ public final class EnrichPolicy implements Writeable, ToXContentFragment {
public void writeTo(StreamOutput out) throws IOException {
out.writeString(type);
out.writeOptionalWriteable(query);
out.writeString(indexPattern);
out.writeStringCollection(indices);
out.writeString(enrichKey);
out.writeStringCollection(enrichValues);
out.writeString(schedule);
@ -152,7 +152,7 @@ public final class EnrichPolicy implements Writeable, ToXContentFragment {
if (query != null) {
builder.field(QUERY.getPreferredName(), query.getQueryAsMap());
}
builder.field(INDEX_PATTERN.getPreferredName(), indexPattern);
builder.array(INDICES.getPreferredName(), indices.toArray(new String[0]));
builder.field(ENRICH_KEY.getPreferredName(), enrichKey);
builder.array(ENRICH_VALUES.getPreferredName(), enrichValues.toArray(new String[0]));
builder.field(SCHEDULE.getPreferredName(), schedule);
@ -166,7 +166,7 @@ public final class EnrichPolicy implements Writeable, ToXContentFragment {
EnrichPolicy policy = (EnrichPolicy) o;
return type.equals(policy.type) &&
Objects.equals(query, policy.query) &&
indexPattern.equals(policy.indexPattern) &&
indices.equals(policy.indices) &&
enrichKey.equals(policy.enrichKey) &&
enrichValues.equals(policy.enrichValues) &&
schedule.equals(policy.schedule);
@ -177,7 +177,7 @@ public final class EnrichPolicy implements Writeable, ToXContentFragment {
return Objects.hash(
type,
query,
indexPattern,
indices,
enrichKey,
enrichValues,
schedule
@ -245,7 +245,7 @@ public final class EnrichPolicy implements Writeable, ToXContentFragment {
(String) args[0],
new EnrichPolicy((String) args[1],
(QuerySource) args[2],
(String) args[3],
(List<String>) args[3],
(String) args[4],
(List<String>) args[5],
(String) args[6])

2
x-pack/plugin/enrich/qa/rest/src/test/java/org/elasticsearch/xpack/enrich/EnrichIT.java
View File

@ -41,7 +41,7 @@ public class EnrichIT extends ESRestTestCase {
public void testBasicFlow() throws Exception {
// Create the policy:
Request putPolicyRequest = new Request("PUT", "/_enrich/policy/my_policy");
putPolicyRequest.setJsonEntity("{\"type\": \"exact_match\",\"index_pattern\": \"my-index*\", \"enrich_key\": \"host\", " +
putPolicyRequest.setJsonEntity("{\"type\": \"exact_match\",\"indices\": [\"my-index*\"], \"enrich_key\": \"host\", " +
"\"enrich_values\": [\"globalRank\", \"tldRank\", \"tld\"], \"schedule\": \"0 5 * * *\"}");
assertOK(client().performRequest(putPolicyRequest));

4
x-pack/plugin/enrich/qa/rest/src/test/resources/rest-api-spec/test/enrich/10_basic.yml
View File

@ -6,7 +6,7 @@
name: policy-crud
body:
type: exact_match
index_pattern: "bar*"
indices: ["bar*"]
enrich_key: baz
enrich_values: ["a", "b"]
schedule: "*/120"
@ -17,7 +17,7 @@
- length: { policies: 1 }
- match: { policies.0.name: policy-crud }
- match: { policies.0.type: exact_match }
- match: { policies.0.index_pattern: "bar*" }
- match: { policies.0.indices: ["bar*"] }
- match: { policies.0.enrich_key: baz }
- match: { policies.0.enrich_values: ["a", "b"] }
- match: { policies.0.schedule: "*/120" }

10
x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPolicyRunner.java
View File

@ -73,9 +73,9 @@ public class EnrichPolicyRunner implements Runnable {
public void run() {
// Collect the source index information
logger.info("Policy [{}]: Running enrich policy", policyName);
final String sourceIndexPattern = policy.getIndexPattern();
logger.debug("Policy [{}]: Checking source index [{}]", policyName, sourceIndexPattern);
GetIndexRequest getIndexRequest = new GetIndexRequest().indices(sourceIndexPattern);
final String[] sourceIndices = policy.getIndices().toArray(new String[0]);
logger.debug("Policy [{}]: Checking source indices [{}]", policyName, sourceIndices);
GetIndexRequest getIndexRequest = new GetIndexRequest().indices(sourceIndices);
client.admin().indices().getIndex(getIndexRequest, new ActionListener<GetIndexResponse>() {
@Override
public void onResponse(GetIndexResponse getIndexResponse) {
@ -108,7 +108,7 @@ public class EnrichPolicyRunner implements Runnable {
listener.onFailure(
new ElasticsearchException(
"Enrich policy execution for [{}] failed. Could not read mapping for source [{}] included by pattern [{}]",
policyName, sourceIndex, policy.getIndexPattern()));
policyName, sourceIndex, policy.getIndices()));
}
if (properties.contains(policy.getEnrichKey()) == false) {
listener.onFailure(
@ -187,7 +187,7 @@ public class EnrichPolicyRunner implements Runnable {
}
ReindexRequest reindexRequest = new ReindexRequest()
.setDestIndex(destinationIndexName)
.setSourceIndices(policy.getIndexPattern());
.setSourceIndices(policy.getIndices().toArray(new String[0]));
reindexRequest.getSearchRequest().source(searchSourceBuilder);
reindexRequest.getDestination().source(new BytesArray(new byte[0]), XContentType.SMILE);
client.execute(ReindexAction.INSTANCE, reindexRequest, new ActionListener<BulkByScrollResponse>() {

6
x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichPolicyRunnerTests.java
View File

@ -87,7 +87,8 @@ public class EnrichPolicyRunnerTests extends ESSingleNodeTestCase {
List<String> enrichFields = new ArrayList<>();
enrichFields.add("field2");
enrichFields.add("field5");
EnrichPolicy policy = new EnrichPolicy(EnrichPolicy.EXACT_MATCH_TYPE, null, sourceIndex, "field1", enrichFields, "");
EnrichPolicy policy =
new EnrichPolicy(EnrichPolicy.EXACT_MATCH_TYPE, null, Collections.singletonList(sourceIndex), "field1", enrichFields, "");
String policyName = "test1";
ActionListener<PolicyExecutionResult> listener = new ActionListener<PolicyExecutionResult>() {
@ -202,7 +203,8 @@ public class EnrichPolicyRunnerTests extends ESSingleNodeTestCase {
enrichFields.add("idx");
enrichFields.add("field2");
enrichFields.add("field5");
EnrichPolicy policy = new EnrichPolicy(EnrichPolicy.EXACT_MATCH_TYPE, null, sourceIndexPattern, "field1", enrichFields, "");
EnrichPolicy policy = new EnrichPolicy(EnrichPolicy.EXACT_MATCH_TYPE, null, Collections.singletonList(sourceIndexPattern),
"field1", enrichFields, "");
String policyName = "test1";
ActionListener<PolicyExecutionResult> listener = new ActionListener<PolicyExecutionResult>() {

4
x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichPolicyTests.java
View File

@ -59,7 +59,7 @@ public class EnrichPolicyTests extends AbstractSerializingTestCase<EnrichPolicy>
return new EnrichPolicy(
randomFrom(EnrichPolicy.SUPPORTED_POLICY_TYPES),
randomBoolean() ? querySource : null,
randomAlphaOfLength(4),
Arrays.asList(generateRandomStringArray(8, 4, false, false)),
randomAlphaOfLength(4),
Arrays.asList(generateRandomStringArray(8, 4, false, false)),
randomAlphaOfLength(4)
@ -89,7 +89,7 @@ public class EnrichPolicyTests extends AbstractSerializingTestCase<EnrichPolicy>
} else {
assertThat(expectedInstance.getQuery(), nullValue());
}
assertThat(newInstance.getIndexPattern(), equalTo(expectedInstance.getIndexPattern()));
assertThat(newInstance.getIndices(), equalTo(expectedInstance.getIndices()));
assertThat(newInstance.getEnrichKey(), equalTo(expectedInstance.getEnrichKey()));
assertThat(newInstance.getEnrichValues(), equalTo(expectedInstance.getEnrichValues()));
assertThat(newInstance.getSchedule(), equalTo(expectedInstance.getSchedule()));

12
x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichProcessorFactoryTests.java
View File

@ -30,7 +30,8 @@ public class EnrichProcessorFactoryTests extends ESTestCase {
public void testCreateProcessorInstance() throws Exception {
List<String> enrichValues = Arrays.asList("globalRank", "tldRank", "tld");
EnrichPolicy policy = new EnrichPolicy(EnrichPolicy.EXACT_MATCH_TYPE, null, "source_index", "my_key", enrichValues, "schedule");
EnrichPolicy policy = new EnrichPolicy(EnrichPolicy.EXACT_MATCH_TYPE, null, Collections.singletonList("source_index"), "my_key",
enrichValues, "schedule");
EnrichProcessorFactory factory = new EnrichProcessorFactory(createClusterStateSupplier("majestic", policy), null);
Map<String, Object> config = new HashMap<>();
@ -103,7 +104,8 @@ public class EnrichProcessorFactoryTests extends ESTestCase {
public void testPolicyNameMissing() {
List<String> enrichValues = Arrays.asList("globalRank", "tldRank", "tld");
EnrichPolicy policy = new EnrichPolicy(EnrichPolicy.EXACT_MATCH_TYPE, null, "source_index", "my_key", enrichValues, "schedule");
EnrichPolicy policy = new EnrichPolicy(EnrichPolicy.EXACT_MATCH_TYPE, null, Collections.singletonList("source_index"), "my_key",
enrichValues, "schedule");
EnrichProcessorFactory factory = new EnrichProcessorFactory(createClusterStateSupplier("_name", policy), null);
Map<String, Object> config = new HashMap<>();
@ -134,7 +136,8 @@ public class EnrichProcessorFactoryTests extends ESTestCase {
public void testUnsupportedPolicy() {
List<String> enrichValues = Arrays.asList("globalRank", "tldRank", "tld");
EnrichPolicy policy = new EnrichPolicy("unsupported", null, "source_index", "my_key", enrichValues, "schedule");
EnrichPolicy policy =
new EnrichPolicy("unsupported", null, Collections.singletonList("source_index"), "my_key", enrichValues, "schedule");
EnrichProcessorFactory factory = new EnrichProcessorFactory(createClusterStateSupplier("majestic", policy), null);
Map<String, Object> config = new HashMap<>();
@ -166,7 +169,8 @@ public class EnrichProcessorFactoryTests extends ESTestCase {
public void testNonExistingDecorateField() throws Exception {
List<String> enrichValues = Arrays.asList("globalRank", "tldRank", "tld");
EnrichPolicy policy = new EnrichPolicy(EnrichPolicy.EXACT_MATCH_TYPE, null, "source_index", "my_key", enrichValues, "schedule");
EnrichPolicy policy = new EnrichPolicy(EnrichPolicy.EXACT_MATCH_TYPE, null, Collections.singletonList("source_index"), "my_key",
enrichValues, "schedule");
EnrichProcessorFactory factory = new EnrichProcessorFactory(createClusterStateSupplier("majestic", policy), null);
Map<String, Object> config = new HashMap<>();

25
x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/ExactMatchProcessorTests.java
View File

@ -53,8 +53,8 @@ public class ExactMatchProcessorTests extends ESTestCase {
indexWriter.addDocument(createEnrichDocument("eops.nl", "globalRank", 4567, "tldRank", 80, "tld", "nl"));
indexWriter.commit();
EnrichPolicy policy = new EnrichPolicy(EnrichPolicy.EXACT_MATCH_TYPE, null, "majestic_index", "key",
Collections.emptyList(), "schedule");
EnrichPolicy policy = new EnrichPolicy(EnrichPolicy.EXACT_MATCH_TYPE, null,
Collections.singletonList("majestic_index"), "key", Collections.emptyList(), "schedule");
Function<String, EnrichPolicy> policyLookup = policyName -> policy;
try (IndexReader indexReader = DirectoryReader.open(directory)) {
@ -101,8 +101,8 @@ public class ExactMatchProcessorTests extends ESTestCase {
indexWriter.addDocument(createEnrichDocument("eops.nl", "globalRank", 4567, "tldRank", 80, "tld", "nl"));
indexWriter.commit();
EnrichPolicy policy = new EnrichPolicy(EnrichPolicy.EXACT_MATCH_TYPE, null, "majestic_index", "key",
Collections.emptyList(), "schedule");
EnrichPolicy policy = new EnrichPolicy(EnrichPolicy.EXACT_MATCH_TYPE, null, Collections.singletonList("majestic_index"),
"key", Collections.emptyList(), "schedule");
Function<String, EnrichPolicy> policyLookup = policyName -> policy;
try (IndexReader indexReader = DirectoryReader.open(directory)) {
@ -133,8 +133,8 @@ public class ExactMatchProcessorTests extends ESTestCase {
indexWriter.addDocument(createEnrichDocument("eops.nl", "globalRank", 4567, "tldRank", 80, "tld", "nl"));
indexWriter.commit();
EnrichPolicy policy = new EnrichPolicy(EnrichPolicy.EXACT_MATCH_TYPE, null, "majestic_index", "key",
Collections.emptyList(), "schedule");
EnrichPolicy policy = new EnrichPolicy(EnrichPolicy.EXACT_MATCH_TYPE, null, Collections.singletonList("majestic_index"),
"key", Collections.emptyList(), "schedule");
Function<String, EnrichPolicy> policyLookup = policyName -> policy;
try (IndexReader indexReader = DirectoryReader.open(directory)) {
@ -161,8 +161,8 @@ public class ExactMatchProcessorTests extends ESTestCase {
try (IndexWriter indexWriter = new IndexWriter(directory, iwConfig)) {
indexWriter.commit();
EnrichPolicy policy = new EnrichPolicy(EnrichPolicy.EXACT_MATCH_TYPE, null, "majestic_index", "key",
Collections.emptyList(), "schedule");
EnrichPolicy policy = new EnrichPolicy(EnrichPolicy.EXACT_MATCH_TYPE, null, Collections.singletonList("majestic_index"),
"key", Collections.emptyList(), "schedule");
Function<String, EnrichPolicy> policyLookup = policyName -> policy;
try (IndexReader indexReader = DirectoryReader.open(directory)) {
@ -193,8 +193,8 @@ public class ExactMatchProcessorTests extends ESTestCase {
indexWriter.addDocument(document);
indexWriter.commit();
EnrichPolicy policy = new EnrichPolicy(EnrichPolicy.EXACT_MATCH_TYPE, null, "majestic_index", "key",
Collections.emptyList(), "schedule");
EnrichPolicy policy = new EnrichPolicy(EnrichPolicy.EXACT_MATCH_TYPE, null, Collections.singletonList("majestic_index"),
"key", Collections.emptyList(), "schedule");
Function<String, EnrichPolicy> policyLookup = policyName -> policy;
try (IndexReader indexReader = DirectoryReader.open(directory)) {
@ -224,8 +224,9 @@ public class ExactMatchProcessorTests extends ESTestCase {
}
public void testIgnoreKeyMissing() throws Exception {
EnrichPolicy policy = new EnrichPolicy(EnrichPolicy.EXACT_MATCH_TYPE, null, "majestic_index", "key",
Collections.emptyList(), "schedule");
EnrichPolicy policy = new EnrichPolicy(EnrichPolicy.EXACT_MATCH_TYPE, null, Collections.singletonList("majestic_index"), "key",
Collections.emptyList(),
"schedule");
Function<String, EnrichPolicy> policyLookup = policyName -> policy;
{
ExactMatchProcessor processor = new ExactMatchProcessor("_tag", policyLookup, indexExpression -> null, "_name", "domain",