honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

EQL: make allow_no_indices true by default (#63573) (#63645) 

* Allow all indices options variants
Irrespective of allow_no_indices value, throw VerificationException when
there is no index validated

Co-authored-by: Andrei Stefan <astefan@users.noreply.github.com>
This commit is contained in:
Ryland Herrick 2020-10-13 19:41:04 -05:00 committed by GitHub
parent 9015b50e1b
commit 7e8769a666
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 179 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
client/rest-high-level/src/main/java/org/elasticsearch/client/eql/EqlSearchRequest.java
View File

@ -34,7 +34,7 @@ import java.util.Objects;
public class EqlSearchRequest implements Validatable, ToXContentObject {
private String[] indices;
private IndicesOptions indicesOptions = IndicesOptions.fromOptions(true, false, true, false);
private IndicesOptions indicesOptions = IndicesOptions.fromOptions(true, true, true, false);
private QueryBuilder filter = null;
private String timestampField = "@timestamp";

112
x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlRestValidationTestCase.java Normal file
View File

@ -0,0 +1,112 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.test.eql;
import org.elasticsearch.client.Request;
import org.elasticsearch.client.Response;
import org.elasticsearch.client.ResponseException;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.test.rest.ESRestTestCase;
import org.junit.Before;
import java.io.IOException;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.elasticsearch.xpack.ql.util.StringUtils.EMPTY;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo;
public abstract class EqlRestValidationTestCase extends ESRestTestCase {
// TODO: handle for existent indices the patterns "test,inexistent", "inexistent,test" that seem to work atm as is
private static final String[] existentIndexName = new String[] {"test,inexistent*", "test*,inexistent*", "inexistent*,test"};
private static final String[] inexistentIndexName = new String[] {"inexistent", "inexistent*", "inexistent1*,inexistent2*"};
@Before
public void prepareIndices() throws IOException {
createIndex("test", Settings.EMPTY);
Object[] fieldsAndValues = new Object[] {"event_type", "my_event", "@timestamp", "2020-10-08T12:35:48Z", "val", 0};
XContentBuilder document = jsonBuilder().startObject();
for (int i = 0; i < fieldsAndValues.length; i += 2) {
document.field((String) fieldsAndValues[i], fieldsAndValues[i + 1]);
}
document.endObject();
final Request request = new Request("POST", "/test/_doc/" + 0);
request.setJsonEntity(Strings.toString(document));
assertOK(client().performRequest(request));
assertOK(adminClient().performRequest(new Request("POST", "/test/_refresh")));
}
public void testDefaultIndicesOptions() throws IOException {
String message = "\"root_cause\":[{\"type\":\"verification_exception\",\"reason\":\"Found 1 problem\\nline -1:-1: Unknown index";
assertErrorMessageOnInexistentIndices(EMPTY, true, message, EMPTY);
assertErrorMessageOnExistentIndices("?allow_no_indices=false", false, message, EMPTY);
assertValidRequestOnExistentIndices(EMPTY);
}
public void testAllowNoIndicesOption() throws IOException {
boolean allowNoIndices = randomBoolean();
boolean setAllowNoIndices = randomBoolean();
boolean isAllowNoIndices = allowNoIndices || setAllowNoIndices == false;
String allowNoIndicesTrueMessage = "\"root_cause\":[{\"type\":\"verification_exception\",\"reason\":"
+ "\"Found 1 problem\\nline -1:-1: Unknown index";
String allowNoIndicesFalseMessage = "\"root_cause\":[{\"type\":\"index_not_found_exception\",\"reason\":\"no such index";
String reqParameter = setAllowNoIndices ? "?allow_no_indices=" + allowNoIndices : EMPTY;
assertErrorMessageOnInexistentIndices(reqParameter, isAllowNoIndices, allowNoIndicesTrueMessage, allowNoIndicesFalseMessage);
if (isAllowNoIndices) {
assertValidRequestOnExistentIndices(reqParameter);
}
}
private void assertErrorMessageOnExistentIndices(String reqParameter, boolean isAllowNoIndices, String allowNoIndicesTrueMessage,
String allowNoIndicesFalseMessage) throws IOException {
assertErrorMessages(existentIndexName, reqParameter, isAllowNoIndices, allowNoIndicesTrueMessage, allowNoIndicesFalseMessage);
}
private void assertErrorMessageOnInexistentIndices(String reqParameter, boolean isAllowNoIndices, String allowNoIndicesTrueMessage,
String allowNoIndicesFalseMessage) throws IOException {
assertErrorMessages(inexistentIndexName, reqParameter, isAllowNoIndices, allowNoIndicesTrueMessage, allowNoIndicesFalseMessage);
}
private void assertErrorMessages(String[] indices, String reqParameter, boolean isAllowNoIndices, String allowNoIndicesTrueMessage,
String allowNoIndicesFalseMessage) throws IOException {
for (String indexName : indices) {
final Request request = createRequest(indexName, reqParameter);
ResponseException exc = expectThrows(ResponseException.class, () -> client().performRequest(request));
assertThat(exc.getResponse().getStatusLine().getStatusCode(), equalTo(isAllowNoIndices ? 400 : 404));
// TODO add the index name to the message to be checked. Waiting on https://github.com/elastic/elasticsearch/issues/63529
assertThat(exc.getMessage(), containsString(isAllowNoIndices ? allowNoIndicesTrueMessage : allowNoIndicesFalseMessage));
}
}
private Request createRequest(String indexName, String reqParameter) throws IOException {
final Request request = new Request("POST", "/" + indexName + "/_eql/search" + reqParameter);
request.setJsonEntity(Strings.toString(JsonXContent.contentBuilder()
.startObject()
.field("event_category_field", "event_type")
.field("query", "my_event where true")
.endObject()));
return request;
}
private void assertValidRequestOnExistentIndices(String reqParameter) throws IOException {
for (String indexName : existentIndexName) {
final Request request = createRequest(indexName, reqParameter);
Response response = client().performRequest(request);
assertOK(response);
}
}
}

7
x-pack/plugin/eql/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlRestValidationIT.java Normal file
View File

@ -0,0 +1,7 @@
package org.elasticsearch.xpack.eql;
import org.elasticsearch.test.eql.EqlRestValidationTestCase;
public class EqlRestValidationIT extends EqlRestValidationTestCase {
}

2
x-pack/plugin/eql/qa/security/src/javaRestTest/java/org/elasticsearch/xpack/eql/AsyncEqlSecurityIT.java
View File

@ -89,7 +89,7 @@ public class AsyncEqlSecurityIT extends ESRestTestCase {
}
ResponseException exc = expectThrows(ResponseException.class,
() -> submitAsyncEqlSearch("index-" + other, "*", TimeValue.timeValueSeconds(10), user));
assertThat(exc.getResponse().getStatusLine().getStatusCode(), equalTo(404));
assertThat(exc.getResponse().getStatusLine().getStatusCode(), equalTo(400));
}
static String extractResponseId(Response response) throws IOException {

14
x-pack/plugin/eql/qa/security/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlRestValidationIT.java Normal file
View File

@ -0,0 +1,14 @@
package org.elasticsearch.xpack.eql;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.test.eql.EqlRestValidationTestCase;
import static org.elasticsearch.xpack.eql.SecurityUtils.secureClientSettings;
public class EqlRestValidationIT extends EqlRestValidationTestCase {
@Override
protected Settings restClientSettings() {
return secureClientSettings();
}
}

8
x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchRequest.java
View File

@ -40,8 +40,7 @@ public class EqlSearchRequest extends ActionRequest implements IndicesRequest.Re
public static TimeValue DEFAULT_KEEP_ALIVE = TimeValue.timeValueDays(5);
private String[] indices;
private IndicesOptions indicesOptions = IndicesOptions.fromOptions(true,
false, true, false);
private IndicesOptions indicesOptions = IndicesOptions.fromOptions(true, true, true, false);
private QueryBuilder filter = null;
private String timestampField = FIELD_TIMESTAMP;
@ -122,13 +121,8 @@ public class EqlSearchRequest extends ActionRequest implements IndicesRequest.Re
if (indicesOptions == null) {
validationException = addValidationError("indicesOptions is null", validationException);
} else {
if (indicesOptions.allowNoIndices()) {
validationException = addValidationError("allowNoIndices must be false", validationException);
}
}
if (query == null || query.isEmpty()) {
validationException = addValidationError("query is null or empty", validationException);
}

7
x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/analysis/PreAnalyzer.java
View File

@ -6,14 +6,21 @@
package org.elasticsearch.xpack.eql.analysis;
import org.elasticsearch.xpack.ql.common.Failure;
import org.elasticsearch.xpack.ql.index.IndexResolution;
import org.elasticsearch.xpack.ql.plan.logical.EsRelation;
import org.elasticsearch.xpack.ql.plan.logical.LogicalPlan;
import org.elasticsearch.xpack.ql.plan.logical.UnresolvedRelation;
import java.util.Collections;
public class PreAnalyzer {
public LogicalPlan preAnalyze(LogicalPlan plan, IndexResolution indices) {
// wrap a potential index_not_found_exception with a VerificationException (expected by client)
if (indices.isValid() == false) {
throw new VerificationException(Collections.singletonList(Failure.fail(plan, indices.toString())));
}
if (plan.analyzed() == false) {
final EsRelation esRelation = new EsRelation(plan.source(), indices.get(), false);
// FIXME: includeFrozen needs to be set already

5
x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlSearchAction.java
View File

@ -108,7 +108,6 @@ public class TransportEqlSearchAction extends HandledTransportAction<EqlSearchRe
ZoneId zoneId = DateUtils.of("Z");
QueryBuilder filter = request.filter();
TimeValue timeout = TimeValue.timeValueSeconds(30);
boolean includeFrozen = request.indicesOptions().ignoreThrottled() == false;
String clientId = null;
ParserParams params = new ParserParams(zoneId)
@ -118,8 +117,8 @@ public class TransportEqlSearchAction extends HandledTransportAction<EqlSearchRe
.size(request.size())
.fetchSize(request.fetchSize());
EqlConfiguration cfg = new EqlConfiguration(request.indices(), zoneId, username, clusterName, filter, timeout, includeFrozen,
request.fetchSize(), clientId, new TaskId(nodeId, task.getId()), task);
EqlConfiguration cfg = new EqlConfiguration(request.indices(), zoneId, username, clusterName, filter, timeout,
request.indicesOptions(), request.fetchSize(), clientId, new TaskId(nodeId, task.getId()), task);
planExecutor.eql(cfg, request.query(), params, wrap(r -> listener.onResponse(createResponse(r, task.getExecutionId())),
listener::onFailure));
}

11
x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EqlConfiguration.java
View File

@ -6,6 +6,7 @@
package org.elasticsearch.xpack.eql.session;
import org.elasticsearch.action.support.IndicesOptions;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.unit.TimeValue;
@ -20,7 +21,7 @@ public class EqlConfiguration extends org.elasticsearch.xpack.ql.session.Configu
private final String[] indices;
private final TimeValue requestTimeout;
private final String clientId;
private final boolean includeFrozenIndices;
private final IndicesOptions indicesOptions;
private final TaskId taskId;
private final EqlSearchTask task;
private final int fetchSize;
@ -29,7 +30,7 @@ public class EqlConfiguration extends org.elasticsearch.xpack.ql.session.Configu
private final QueryBuilder filter;
public EqlConfiguration(String[] indices, ZoneId zi, String username, String clusterName, QueryBuilder filter, TimeValue requestTimeout,
boolean includeFrozen, int fetchSize, String clientId, TaskId taskId,
IndicesOptions indicesOptions, int fetchSize, String clientId, TaskId taskId,
EqlSearchTask task) {
super(zi, username, clusterName);
@ -37,7 +38,7 @@ public class EqlConfiguration extends org.elasticsearch.xpack.ql.session.Configu
this.filter = filter;
this.requestTimeout = requestTimeout;
this.clientId = clientId;
this.includeFrozenIndices = includeFrozen;
this.indicesOptions = indicesOptions;
this.taskId = taskId;
this.task = task;
this.fetchSize = fetchSize;
@ -67,8 +68,8 @@ public class EqlConfiguration extends org.elasticsearch.xpack.ql.session.Configu
return clientId;
}
public boolean includeFrozen() {
return includeFrozenIndices;
public IndicesOptions indicesOptions() {
return indicesOptions;
}
public boolean isCancelled() {

2
x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EqlSession.java
View File

@ -100,7 +100,7 @@ public class EqlSession {
listener.onFailure(new TaskCancelledException("cancelled"));
return;
}
indexResolver.resolveAsMergedMapping(indexWildcard, null, configuration.includeFrozen(), configuration.filter(),
indexResolver.resolveAsMergedMapping(indexWildcard, null, configuration.indicesOptions(), configuration.filter(),
map(listener, r -> preAnalyzer.preAnalyze(parsed, r))
);
}

10
x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/EqlTestUtils.java
View File

@ -6,6 +6,7 @@
package org.elasticsearch.xpack.eql;
import org.elasticsearch.action.support.IndicesOptions;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.tasks.TaskId;
import org.elasticsearch.xpack.core.async.AsyncExecutionId;
@ -32,7 +33,7 @@ public final class EqlTestUtils {
}
public static final EqlConfiguration TEST_CFG = new EqlConfiguration(new String[] {"none"},
org.elasticsearch.xpack.ql.util.DateUtils.UTC, "nobody", "cluster", null, TimeValue.timeValueSeconds(30), false,
org.elasticsearch.xpack.ql.util.DateUtils.UTC, "nobody", "cluster", null, TimeValue.timeValueSeconds(30), null,
123, "", new TaskId("test", 123), null);
public static EqlConfiguration randomConfiguration() {
@ -42,7 +43,7 @@ public final class EqlTestUtils {
randomAlphaOfLength(16),
null,
new TimeValue(randomNonNegativeLong()),
randomBoolean(),
randomIndicesOptions(),
randomIntBetween(1, 1000),
randomAlphaOfLength(16),
new TaskId(randomAlphaOfLength(10), randomNonNegativeLong()),
@ -62,4 +63,9 @@ public final class EqlTestUtils {
return new InsensitiveNotEquals(EMPTY, left, right, randomZone());
}
public static IndicesOptions randomIndicesOptions() {
return IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(),
randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean());
}
}

21
x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java
View File

@ -277,6 +277,18 @@ public class IndexResolver {
listener.onResponse(result);
}
/**
* Resolves a pattern to one (potentially compound meaning that spawns multiple indices) mapping.
*/
public void resolveAsMergedMapping(String indexWildcard, String javaRegex, IndicesOptions indicesOptions, QueryBuilder filter,
ActionListener<IndexResolution> listener) {
FieldCapabilitiesRequest fieldRequest = createFieldCapsRequest(indexWildcard, indicesOptions, filter);
client.fieldCaps(fieldRequest,
ActionListener.wrap(
response -> listener.onResponse(mergedMappings(typeRegistry, indexWildcard, response.getIndices(), response.get())),
listener::onFailure));
}
/**
* Resolves a pattern to one (potentially compound meaning that spawns multiple indices) mapping.
*/
@ -458,7 +470,7 @@ public class IndexResolver {
return new EsField(fieldName, esType, props, isAggregateable, isAlias);
}
private static FieldCapabilitiesRequest createFieldCapsRequest(String index, boolean includeFrozen, QueryBuilder filter) {
private static FieldCapabilitiesRequest createFieldCapsRequest(String index, IndicesOptions indicesOptions, QueryBuilder filter) {
return new FieldCapabilitiesRequest()
.indices(Strings.commaDelimitedListToStringArray(index))
.fields("*")
@ -466,7 +478,12 @@ public class IndexResolver {
.indexFilter(filter)
//lenient because we throw our own errors looking at the response e.g. if something was not resolved
//also because this way security doesn't throw authorization exceptions but rather honors ignore_unavailable
.indicesOptions(includeFrozen ? FIELD_CAPS_FROZEN_INDICES_OPTIONS : FIELD_CAPS_INDICES_OPTIONS);
.indicesOptions(indicesOptions);
}
private static FieldCapabilitiesRequest createFieldCapsRequest(String index, boolean includeFrozen, QueryBuilder filter) {
IndicesOptions indicesOptions = includeFrozen ? FIELD_CAPS_FROZEN_INDICES_OPTIONS : FIELD_CAPS_INDICES_OPTIONS;
return createFieldCapsRequest(index, indicesOptions, filter);
}
/**