Allow _cat indices & aliases to use indices options (#53248)

This commit adjusts the _cat/indices and _cat/aliases APIs to allow
specifying indices options, so that these APIs can handle hidden
indices/aliases in the same way as other APIs.

Also adds the hidden option to the expand_wildcards parameter
in the YAML spec for every API that accepts it.
This commit is contained in:
Gordon Brown 2020-03-16 11:25:05 -06:00 committed by GitHub
parent 7571ca437a
commit 031932b32f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
42 changed files with 602 additions and 9 deletions

View File

@ -53,6 +53,18 @@
"type":"boolean",
"description":"Verbose mode. Display column headers",
"default":false
},
"expand_wildcards":{
"type":"enum",
"options":[
"open",
"closed",
"hidden",
"none",
"all"
],
"default": ["all"],
"description":"Whether to expand wildcard expression to concrete indices that are open, closed or both."
}
}
}

View File

@ -107,6 +107,18 @@
"type":"boolean",
"description":"If set to true segment stats will include stats for segments that are not currently loaded into memory",
"default":false
},
"expand_wildcards":{
"type":"enum",
"options":[
"open",
"closed",
"hidden",
"none",
"all"
],
"default": "all",
"description":"Whether to expand wildcard expression to concrete indices that are open, closed or both."
}
}
}

View File

@ -33,6 +33,7 @@
"options":[
"open",
"closed",
"hidden",
"none",
"all"
],

View File

@ -97,6 +97,7 @@
"options":[
"open",
"closed",
"hidden",
"none",
"all"
],

View File

@ -69,6 +69,7 @@
"options":[
"open",
"closed",
"hidden",
"none",
"all"
],

View File

@ -79,10 +79,16 @@
"description" : "What to do when the delete by query hits version conflicts?"
},
"expand_wildcards": {
"type" : "enum",
"options" : ["open","closed","none","all"],
"default" : "open",
"description" : "Whether to expand wildcard expression to concrete indices that are open, closed or both."
"type" : "enum",
"options" : [
"open",
"closed",
"hidden",
"none",
"all"
],
"default" : "open",
"description" :"Whether to expand wildcard expression to concrete indices that are open, closed or both."
},
"lenient": {
"type" : "boolean",

View File

@ -47,6 +47,7 @@
"options":[
"open",
"closed",
"hidden",
"none",
"all"
],

View File

@ -53,6 +53,7 @@
"options":[
"open",
"closed",
"hidden",
"none",
"all"
],

View File

@ -43,6 +43,7 @@
"options":[
"open",
"closed",
"hidden",
"none",
"all"
],

View File

@ -43,6 +43,7 @@
"options":[
"open",
"closed",
"hidden",
"none",
"all"
],

View File

@ -39,6 +39,7 @@
"options":[
"open",
"closed",
"hidden",
"none",
"all"
],

View File

@ -51,6 +51,7 @@
"options":[
"open",
"closed",
"hidden",
"none",
"all"
],

View File

@ -39,6 +39,7 @@
"options":[
"open",
"closed",
"hidden",
"none",
"all"
],

View File

@ -51,6 +51,7 @@
"options":[
"open",
"closed",
"hidden",
"none",
"all"
],

View File

@ -45,6 +45,7 @@
"options":[
"open",
"closed",
"hidden",
"none",
"all"
],

View File

@ -43,6 +43,7 @@
"options":[
"open",
"closed",
"hidden",
"none",
"all"
],

View File

@ -69,10 +69,11 @@
"options":[
"open",
"closed",
"hidden",
"none",
"all"
],
"default":"all",
"default": ["all"],
"description":"Whether to expand wildcard expression to concrete indices that are open, closed or both."
},
"local":{

View File

@ -105,6 +105,7 @@
"options":[
"open",
"closed",
"hidden",
"none",
"all"
],

View File

@ -83,6 +83,7 @@
"options":[
"open",
"closed",
"hidden",
"none",
"all"
],

View File

@ -73,6 +73,7 @@
"options":[
"open",
"closed",
"hidden",
"none",
"all"
],

View File

@ -41,6 +41,7 @@
"options":[
"open",
"closed",
"hidden",
"none",
"all"
],

View File

@ -43,6 +43,7 @@
"options":[
"open",
"closed",
"hidden",
"none",
"all"
],

View File

@ -189,6 +189,7 @@
"options":[
"open",
"closed",
"hidden",
"none",
"all"
],

View File

@ -53,6 +53,7 @@
"options":[
"open",
"closed",
"hidden",
"none",
"all"
],

View File

@ -43,6 +43,7 @@
"options":[
"open",
"closed",
"hidden",
"none",
"all"
],

View File

@ -41,6 +41,7 @@
"options":[
"open",
"closed",
"hidden",
"none",
"all"
],

View File

@ -51,6 +51,7 @@
"options":[
"open",
"closed",
"hidden",
"none",
"all"
],

View File

@ -137,6 +137,7 @@
"options":[
"open",
"closed",
"hidden",
"none",
"all"
],

View File

@ -37,6 +37,7 @@
"options":[
"open",
"closed",
"hidden",
"none",
"all"
],

View File

@ -69,6 +69,7 @@
"options":[
"open",
"closed",
"hidden",
"none",
"all"
],

View File

@ -43,6 +43,7 @@
"options":[
"open",
"closed",
"hidden",
"none",
"all"
],

View File

@ -107,6 +107,7 @@
"options":[
"open",
"closed",
"hidden",
"none",
"all"
],

View File

@ -55,6 +55,7 @@
"options":[
"open",
"closed",
"hidden",
"none",
"all"
],

View File

@ -65,6 +65,7 @@
"options":[
"open",
"closed",
"hidden",
"none",
"all"
],

View File

@ -89,6 +89,7 @@
"options":[
"open",
"closed",
"hidden",
"none",
"all"
],

View File

@ -0,0 +1,147 @@
---
"Test cat aliases output with a hidden index with a hidden alias":
- skip:
version: "- 7.6.99"
reason: "hidden indices and aliases were added in 7.7.0"
- do:
indices.create:
index: test
body:
settings:
number_of_shards: "1"
number_of_replicas: "0"
index:
hidden: true
aliases:
test_alias:
is_hidden: true
- do:
cat.aliases: {}
- match:
$body: |
/^
test_alias \s+
test \s+
- \s+
- \s+
- \s+
- \s+
$/
- do:
cat.aliases:
name: test_alias
- match:
$body: |
/^
test_alias \s+
test \s+
- \s+
- \s+
- \s+
- \s+
$/
- do:
cat.aliases:
expand_wildcards: ["open","closed"]
- match:
$body: |
/^
$/
---
"Test cat aliases output with a hidden index with a visible alias":
- skip:
version: "- 7.6.99"
reason: "hidden indices and aliases were added in 7.7.0"
- do:
indices.create:
index: test
body:
settings:
number_of_shards: "1"
number_of_replicas: "0"
index:
hidden: true
aliases:
test_alias: {}
- do:
cat.aliases: {}
- match:
$body: |
/^
test_alias \s+
test \s+
- \s+
- \s+
- \s+
- \s+
$/
- do:
cat.aliases:
name: test_alias
- match:
$body: |
/^
test_alias \s+
test \s+
- \s+
- \s+
- \s+
- \s+
$/
---
"Test cat aliases output with a visible index with a hidden alias":
- skip:
version: "- 7.6.99"
reason: "hidden indices and aliases were added in 7.7.0"
- do:
indices.create:
index: test
body:
settings:
number_of_shards: "1"
number_of_replicas: "0"
aliases:
test_alias:
is_hidden: true
- do:
cat.aliases: {}
- match:
$body: |
/^
test_alias \s+
test \s+
- \s+
- \s+
- \s+
- \s+
$/
- do:
cat.aliases:
name: test_alias
- match:
$body: |
/^
test_alias \s+
test \s+
- \s+
- \s+
- \s+
- \s+
$/

View File

@ -0,0 +1,240 @@
---
"Test cat indices output for hidden index":
- skip:
version: "- 7.6.99"
reason: "hidden indices were added in 7.7.0"
- do:
indices.create:
index: index1
body:
settings:
number_of_shards: "1"
number_of_replicas: "0"
index:
hidden: true
- do:
cat.indices: {}
- match:
$body: |
/^$/
- do:
cat.indices:
expand_wildcards: ["all"]
- match:
$body: |
/^(green \s+
open \s+
index1 \s+
([a-zA-Z0-9=/_+]|[\\\-]){22} \s+
1 \s+
0 \s+
0 \s+
0 \s+
(\d+|\d+[.]\d+)(kb|b) \s+
(\d+|\d+[.]\d+)(kb|b) \s*
)
$/
---
"Test cat indices output for dot-hidden index and dot-prefixed pattern":
- skip:
version: "- 7.6.99"
reason: "hidden indices were added in 7.7.0"
- do:
indices.create:
index: .index1
body:
settings:
number_of_shards: "1"
number_of_replicas: "0"
index:
hidden: true
- do:
cat.indices: {}
- match:
$body: |
/^$/
- do:
cat.indices:
index: ".*"
- match:
$body: |
/^(green \s+
open \s+
\.index1 \s+
([a-zA-Z0-9=/_+]|[\\\-]){22} \s+
1 \s+
0 \s+
0 \s+
0 \s+
(\d+|\d+[.]\d+)(kb|b) \s+
(\d+|\d+[.]\d+)(kb|b) \s*
)
$/
---
"Test cat indices output with a hidden index with a visible alias":
- skip:
version: "- 7.6.99"
reason: "hidden indices were added in 7.7.0"
- do:
indices.create:
index: index1
body:
settings:
number_of_shards: "1"
number_of_replicas: "0"
index:
hidden: true
aliases:
alias1: {}
- do:
cat.indices:
index: "i*"
# Can't use a bare wildcard here because Security replaces wildcards
# it with all matching authorized indices/aliases, including the visible
# alias
- match:
$body: |
/^$/
- do:
cat.indices:
expand_wildcards: ["open", "hidden"]
- match:
$body: |
/^(green \s+
open \s+
index1 \s+
([a-zA-Z0-9=/_+]|[\\\-]){22} \s+
1 \s+
0 \s+
0 \s+
0 \s+
(\d+|\d+[.]\d+)(kb|b) \s+
(\d+|\d+[.]\d+)(kb|b) \s*
)
$/
- do:
cat.indices:
index: alias1
- match:
$body: |
/^(green \s+
open \s+
index1 \s+
([a-zA-Z0-9=/_+]|[\\\-]){22} \s+
1 \s+
0 \s+
0 \s+
0 \s+
(\d+|\d+[.]\d+)(kb|b) \s+
(\d+|\d+[.]\d+)(kb|b) \s*
)
$/
---
"Test cat indices output with a hidden index with a hidden alias":
- skip:
version: "- 7.6.99"
reason: "hidden indices and aliases were added in 7.7.0"
- do:
indices.create:
index: index1
body:
settings:
number_of_shards: "1"
number_of_replicas: "0"
index:
hidden: true
aliases:
alias1:
is_hidden: true
- do:
cat.indices: {}
- match:
$body: |
/^$/
- do:
cat.indices:
expand_wildcards: ["all"]
- match:
$body: |
/^(green \s+
open \s+
index1 \s+
([a-zA-Z0-9=/_+]|[\\\-]){22} \s+
1 \s+
0 \s+
0 \s+
0 \s+
(\d+|\d+[.]\d+)(kb|b) \s+
(\d+|\d+[.]\d+)(kb|b) \s*
)
$/
- do:
cat.indices:
index: alias1
- match:
$body: |
/^(green \s+
open \s+
index1 \s+
([a-zA-Z0-9=/_+]|[\\\-]){22} \s+
1 \s+
0 \s+
0 \s+
0 \s+
(\d+|\d+[.]\d+)(kb|b) \s+
(\d+|\d+[.]\d+)(kb|b) \s*
)
$/
---
"Test cat indices output with a hidden index, dot-hidden alias and dot pattern":
- skip:
version: "- 7.6.99"
reason: "hidden indices and aliases were added in 7.7.0"
- do:
indices.create:
index: index1
body:
settings:
number_of_shards: "1"
number_of_replicas: "0"
index:
hidden: true
aliases:
.alias1:
is_hidden: true
- do:
cat.indices: {}
- match:
$body: |
/^$/
- do:
cat.indices:
index: ".*"
- match:
$body: |
/^(green \s+
open \s+
index1 \s+
([a-zA-Z0-9=/_+]|[\\\-]){22} \s+
1 \s+
0 \s+
0 \s+
0 \s+
(\d+|\d+[.]\d+)(kb|b) \s+
(\d+|\d+[.]\d+)(kb|b) \s*
)
$/

View File

@ -21,6 +21,7 @@ package org.elasticsearch.rest.action.cat;
import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse;
import org.elasticsearch.action.support.IndicesOptions;
import org.elasticsearch.client.node.NodeClient;
import org.elasticsearch.cluster.metadata.AliasMetaData;
import org.elasticsearch.common.Strings;
@ -54,6 +55,7 @@ public class RestAliasAction extends AbstractCatAction {
final GetAliasesRequest getAliasesRequest = request.hasParam("alias") ?
new GetAliasesRequest(Strings.commaDelimitedListToStringArray(request.param("alias"))) :
new GetAliasesRequest();
getAliasesRequest.indicesOptions(IndicesOptions.fromRequest(request, getAliasesRequest.indicesOptions()));
getAliasesRequest.local(request.paramAsBoolean("local", getAliasesRequest.local()));
return channel -> client.admin().indices().getAliases(getAliasesRequest, new RestResponseListener<GetAliasesResponse>(channel) {

View File

@ -91,7 +91,7 @@ public class RestIndicesAction extends AbstractCatAction {
@Override
public RestChannelConsumer doCatRequest(final RestRequest request, final NodeClient client) {
final String[] indices = Strings.splitStringByCommaToArray(request.param("index"));
final IndicesOptions indicesOptions = IndicesOptions.strictExpand();
final IndicesOptions indicesOptions = IndicesOptions.fromRequest(request, IndicesOptions.strictExpand());
final boolean local = request.paramAsBoolean("local", false);
final TimeValue masterNodeTimeout = request.paramAsTime("master_timeout", DEFAULT_MASTER_NODE_TIMEOUT);
final boolean includeUnloadedSegments = request.paramAsBoolean("include_unloaded_segments", false);
@ -110,6 +110,12 @@ public class RestIndicesAction extends AbstractCatAction {
final GroupedActionListener<ActionResponse> groupedListener = createGroupedListener(request, 4, listener);
groupedListener.onResponse(getSettingsResponse);
// The list of indices that will be returned is determined by the indices returned from the Get Settings call.
// All the other requests just provide additional detail, and wildcards may be resolved differently depending on the
// type of request in the presence of security plugins (looking at you, ClusterHealthRequest), so
// force the IndicesOptions for all the sub-requests to be as inclusive as possible.
final IndicesOptions subRequestIndicesOptions = IndicesOptions.lenientExpandHidden();
// Indices that were successfully resolved during the get settings request might be deleted when the subsequent cluster
// state, cluster health and indices stats requests execute. We have to distinguish two cases:
// 1) the deleted index was explicitly passed as parameter to the /_cat/indices request. In this case we want the
@ -118,11 +124,11 @@ public class RestIndicesAction extends AbstractCatAction {
// fail on the deleted index (as we want to ignore wildcards that cannot be resolved).
// This behavior can be ensured by letting the cluster state, cluster health and indices stats requests re-resolve the
// index names with the same indices options that we used for the initial cluster state request (strictExpand).
sendIndicesStatsRequest(indices, indicesOptions, includeUnloadedSegments, client,
sendIndicesStatsRequest(indices, subRequestIndicesOptions, includeUnloadedSegments, client,
ActionListener.wrap(groupedListener::onResponse, groupedListener::onFailure));
sendClusterStateRequest(indices, indicesOptions, local, masterNodeTimeout, client,
sendClusterStateRequest(indices, subRequestIndicesOptions, local, masterNodeTimeout, client,
ActionListener.wrap(groupedListener::onResponse, groupedListener::onFailure));
sendClusterHealthRequest(indices, indicesOptions, local, masterNodeTimeout, client,
sendClusterHealthRequest(indices, subRequestIndicesOptions, local, masterNodeTimeout, client,
ActionListener.wrap(groupedListener::onResponse, groupedListener::onFailure));
}

View File

@ -158,7 +158,9 @@ testClusters.integTest {
keystore 'xpack.security.transport.ssl.secure_key_passphrase', 'testnode'
user username: "x_pack_rest_user", password: "x-pack-test-password"
user username: "cat_test_user", password: "cat-test-password", role: "cat_test_role"
extraConfigFile nodeKey.name, nodeKey
extraConfigFile nodeCert.name, nodeCert
extraConfigFile 'roles.yml', file('src/test/resources/roles.yml')
}

View File

@ -0,0 +1,127 @@
/*
*
* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* * or more contributor license agreements. Licensed under the Elastic License;
* * you may not use this file except in compliance with the Elastic License.
*
*/
package org.elasticsearch.xpack.test.rest;
import org.apache.http.util.EntityUtils;
import org.elasticsearch.client.Request;
import org.elasticsearch.client.Response;
import org.elasticsearch.common.settings.SecureString;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.concurrent.ThreadContext;
import org.elasticsearch.test.SecuritySettingsSourceField;
import org.elasticsearch.test.rest.ESRestTestCase;
import java.io.IOException;
import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
import static org.hamcrest.Matchers.matchesRegex;
public class CatIndicesWithSecurityIT extends ESRestTestCase {
@Override
protected Settings restAdminSettings() {
String token = basicAuthHeaderValue("x_pack_rest_user", SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING);
return Settings.builder()
.put(ThreadContext.PREFIX + ".Authorization", token)
.build();
}
@Override
protected Settings restClientSettings() {
String token = basicAuthHeaderValue("cat_test_user", new SecureString("cat-test-password".toCharArray()));
return Settings.builder()
.put(ThreadContext.PREFIX + ".Authorization", token)
.build();
}
public void testHiddenIndexWithVisibleAlias() throws IOException {
// Create the index and alias
{
final Request createRequest = new Request("PUT", ".index_hidden");
createRequest.setJsonEntity(
"{\"settings\": {\"index.hidden\": true, \"number_of_replicas\": 0}, \"aliases\": {\"index_allowed\": {}}}");
final Response createResponse = adminClient().performRequest(createRequest);
assertOK(createResponse);
ensureGreen("index_allowed");
}
// And it should be visible by a user that has permissions for only the alias
{
final Request catRequest = new Request("GET", "_cat/indices?format=txt");
final Response catResponse = client().performRequest(catRequest);
assertOK(catResponse);
final String resp = EntityUtils.toString(catResponse.getEntity());
assertThat(resp, matchesRegex("(?s)^?green\\s+open\\s+\\.index_hidden.*$"));
}
}
public void testHiddenIndexWithHiddenAlias() throws IOException {
// Create the index and alias
{
final Request createRequest = new Request("PUT", ".index_hidden");
createRequest.setJsonEntity("{\"settings\": {\"index.hidden\": true, \"number_of_replicas\": 0}, "
+ "\"aliases\": {\"index_allowed\": {\"is_hidden\": true}}}");
final Response createResponse = adminClient().performRequest(createRequest);
assertOK(createResponse);
ensureGreen("index_allowed");
}
// It should *not* be visible
{
final Request catRequest = new Request("GET", "_cat/indices?format=txt");
final Response catResponse = client().performRequest(catRequest);
assertOK(catResponse);
final String resp = EntityUtils.toString(catResponse.getEntity());
assertThat(resp, matchesRegex("(?s)\\W*")); // Should have no text
}
// But we should be able to see the index if we ask for hidden things
{
final Request catRequest = new Request("GET", "_cat/indices?format=txt&expand_wildcards=all,hidden");
final Response catResponse = client().performRequest(catRequest);
assertOK(catResponse);
final String resp = EntityUtils.toString(catResponse.getEntity());
assertThat(resp, matchesRegex("(?s)^?green\\s+open\\s+\\.index_hidden.*$"));
}
}
public void testVisibleIndexWithHiddenAlias() throws IOException {
// Create the index and alias
{
final Request createRequest = new Request("PUT", "visible_index");
createRequest.setJsonEntity("{\"settings\": {\"number_of_replicas\": 0}, "
+ "\"aliases\": {\"index_allowed\": {\"is_hidden\": true}}}");
final Response createResponse = adminClient().performRequest(createRequest);
assertOK(createResponse);
ensureGreen("index_allowed");
}
// It should *not* be visible, because this user only has access through the *hidden* alias
{
final Request catRequest = new Request("GET", "_cat/indices?format=txt");
final Response catResponse = client().performRequest(catRequest);
assertOK(catResponse);
final String resp = EntityUtils.toString(catResponse.getEntity());
assertThat(resp, matchesRegex("(?s)\\W*")); // Should have no text
}
// But we should be able to see the index if we ask for hidden things
{
final Request catRequest = new Request("GET", "_cat/indices?format=txt&expand_wildcards=all,hidden");
final Response catResponse = client().performRequest(catRequest);
assertOK(catResponse);
final String resp = EntityUtils.toString(catResponse.getEntity());
assertThat(resp, matchesRegex("(?s)^?green\\s+open\\s+visible_index.*$"));
}
}
}

View File

@ -0,0 +1,7 @@
# Required for CatIndicesWithSecurityIT
cat_test_role:
cluster:
- monitor
indices:
- names: [ "index_allowed" ]
privileges: [ "read", "write", "monitor" ]