honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix date math hidden index resolution (#65278) 

This commit updates the IndexAbstractionResolver so that hidden indices
are properly resolved when date math is in use and when we are checking
if the index is visible.

Closes #65157
Backport of #65236
This commit is contained in:
Jay Modi 2020-11-19 12:40:14 -07:00 committed by GitHub
parent b60d0711b7
commit 893e1a5282
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 77 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstractionResolver.java
View File

@ -146,16 +146,17 @@ public class IndexAbstractionResolver {
return includeDataStreams;
}
assert indexAbstraction.getIndices().size() == 1 : "concrete index must point to a single index";
IndexMetadata indexMetadata = indexAbstraction.getIndices().get(0);
if (isHidden && indicesOptions.expandWildcardsHidden() == false && isVisibleDueToImplicitHidden(expression, index) == false) {
return false;
}
// the index is not hidden and since it is a date math expression, we consider it visible regardless of open/closed
// since it is a date math expression, we consider the index visible regardless of open/closed/hidden as the user is using
// date math to explicitly reference the index
if (dateMathExpression) {
assert IndexMetadata.State.values().length == 2 : "a new IndexMetadata.State value may need to be handled!";
return true;
}
if (isHidden && indicesOptions.expandWildcardsHidden() == false && isVisibleDueToImplicitHidden(expression, index) == false) {
return false;
}
IndexMetadata indexMetadata = indexAbstraction.getIndices().get(0);
if (indexMetadata.getState() == IndexMetadata.State.CLOSE && indicesOptions.expandWildcardsClosed()) {
return true;
}

29
server/src/test/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexTests.java
View File

@ -34,9 +34,13 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
import org.elasticsearch.cluster.metadata.Metadata;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.time.DateFormatter;
import org.elasticsearch.common.util.concurrent.ThreadContext;
import org.elasticsearch.test.ESTestCase;
import java.time.Clock;
import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
@ -46,14 +50,17 @@ import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import static java.util.Arrays.asList;
import static java.util.Collections.singletonList;
import static org.elasticsearch.cluster.DataStreamTestHelper.createTimestampField;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.oneOf;
import static org.hamcrest.core.IsNull.notNullValue;
public class ResolveIndexTests extends ESTestCase {
private final Object[][] indices = new Object[][]{
private final Object[][] indices = new Object[][] {
// name, isClosed, isHidden, isFrozen, dataStream, aliases
{"logs-pgsql-prod-20200101", false, false, true, null, new String[]{"logs-pgsql-prod"}},
{"logs-pgsql-prod-20200102", false, false, true, null, new String[]{"logs-pgsql-prod", "one-off-alias"}},
@ -169,6 +176,26 @@ public class ResolveIndexTests extends ESTestCase {
validateDataStreams(dataStreams, "logs-mysql-test");
}
public void testResolveHiddenProperlyWithDateMath() {
// set up with today's index and following day's index to avoid test failures due to execution time
DateFormatter dateFormatter = DateFormatter.forPattern("uuuu.MM.dd");
Instant now = Instant.now(Clock.systemUTC());
String todaySuffix = dateFormatter.format(now);
String tomorrowSuffix = dateFormatter.format(now.plus(Duration.ofDays(1L)));
Object[][] indices = new Object[][] {
// name, isClosed, isHidden, isFrozen, dataStream, aliases
{"logs-pgsql-prod-" + todaySuffix, false, true, false, null, Strings.EMPTY_ARRAY},
{"logs-pgsql-prod-" + tomorrowSuffix, false, true, false, null, Strings.EMPTY_ARRAY}
};
Metadata metadata = buildMetadata(new Object[][] {}, indices);
String requestedIndex = "<logs-pgsql-prod-{now/d}>";
List<String> resolvedIndices = resolver.resolveIndexAbstractions(singletonList(requestedIndex), IndicesOptions.LENIENT_EXPAND_OPEN,
metadata, asList("logs-pgsql-prod-" + todaySuffix, "logs-pgsql-prod-" + tomorrowSuffix), randomBoolean(), randomBoolean());
assertThat(resolvedIndices.size(), is(1));
assertThat(resolvedIndices.get(0), oneOf("logs-pgsql-prod-" + todaySuffix, "logs-pgsql-prod-" + tomorrowSuffix));
}
private void validateIndices(List<ResolvedIndex> resolvedIndices, String... expectedIndices) {
assertThat(resolvedIndices.size(), equalTo(expectedIndices.length));
for (int k = 0; k < resolvedIndices.size(); k++) {

55
x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java
View File

@ -47,6 +47,7 @@ import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.regex.Regex;
import org.elasticsearch.common.settings.ClusterSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.time.DateFormatter;
import org.elasticsearch.common.util.concurrent.ThreadContext;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.IndexNotFoundException;
@ -74,6 +75,9 @@ import org.joda.time.DateTimeZone;
import org.joda.time.format.DateTimeFormat;
import org.junit.Before;
import java.time.Clock;
import java.time.Duration;
import java.time.Instant;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
@ -97,6 +101,7 @@ import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.hasItem;
import static org.hamcrest.Matchers.hasItems;
import static org.hamcrest.Matchers.not;
import static org.hamcrest.Matchers.oneOf;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.doCallRealMethod;
@ -113,6 +118,8 @@ public class IndicesAndAliasesResolverTests extends ESTestCase {
private IndicesAndAliasesResolver defaultIndicesResolver;
private IndexNameExpressionResolver indexNameExpressionResolver;
private Map<String, RoleDescriptor> roleMap;
private String todaySuffix;
private String tomorrowSuffix;
@Before
public void setup() {
@ -126,6 +133,10 @@ public class IndicesAndAliasesResolverTests extends ESTestCase {
indexNameExpressionResolver = new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY));
DateFormatter dateFormatter = DateFormatter.forPattern("uuuu.MM.dd");
Instant now = Instant.now(Clock.systemUTC());
todaySuffix = dateFormatter.format(now);
tomorrowSuffix = dateFormatter.format(now.plus(Duration.ofDays(1L)));
final boolean withAlias = randomBoolean();
final String securityIndexName = SECURITY_MAIN_ALIAS + (withAlias ? "-" + randomAlphaOfLength(5) : "");
final String dataStreamName = "logs-foobar";
@ -169,6 +180,10 @@ public class IndicesAndAliasesResolverTests extends ESTestCase {
.put(indexBuilder("visible-w-aliases").settings(Settings.builder().put(settings).build())
.putAlias(AliasMetadata.builder("alias-visible").build())
.putAlias(AliasMetadata.builder("alias-visible-mixed").isHidden(false).build()))
.put(indexBuilder("date-hidden-" + todaySuffix)
.settings(Settings.builder().put(settings).put("index.hidden", true).build()))
.put(indexBuilder("date-hidden-" + tomorrowSuffix)
.settings(Settings.builder().put(settings).put("index.hidden", true).build()))
.put(dataStreamIndex1, true)
.put(dataStreamIndex2, true)
.put(dataStreamIndex3, true)
@ -188,7 +203,8 @@ public class IndicesAndAliasesResolverTests extends ESTestCase {
userNoIndices = new User("test", "test");
rolesStore = mock(CompositeRolesStore.class);
String[] authorizedIndices = new String[] { "bar", "bar-closed", "foofoobar", "foobarfoo", "foofoo", "missing", "foofoo-closed",
"hidden-open", "hidden-closed", ".hidden-open", ".hidden-closed"};
"hidden-open", "hidden-closed", ".hidden-open", ".hidden-closed", "date-hidden-" + todaySuffix,
"date-hidden-" + tomorrowSuffix};
String[] dashIndices = new String[]{"-index10", "-index11", "-index20", "-index21"};
roleMap = new HashMap<>();
roleMap.put("role", new RoleDescriptor("role", null,
@ -1042,10 +1058,11 @@ public class IndicesAndAliasesResolverTests extends ESTestCase {
List<String> indices = resolveIndices(request, authorizedIndices).getLocal();
//the union of all resolved indices and aliases gets returned, including hidden indices as Get Aliases includes hidden by default
String[] expectedIndices = new String[]{"bar", "bar-closed", "foofoobar", "foobarfoo", "foofoo", "foofoo-closed", "alias1",
"hidden-open", "hidden-closed", ".hidden-open", ".hidden-closed"};
"hidden-open", "hidden-closed", ".hidden-open", ".hidden-closed", "date-hidden-" + todaySuffix,
"date-hidden-" + tomorrowSuffix};
assertSameValues(indices, expectedIndices);
String[] replacedIndices = new String[]{"bar", "bar-closed", "foofoobar", "foobarfoo", "foofoo", "foofoo-closed", "hidden-open",
"hidden-closed", ".hidden-open", ".hidden-closed"};
"hidden-closed", ".hidden-open", ".hidden-closed", "date-hidden-" + todaySuffix, "date-hidden-" + tomorrowSuffix};
//_all gets replaced with all indices that user is authorized for
assertThat(request.indices(), arrayContainingInAnyOrder(replacedIndices));
assertThat(request.aliases(), arrayContainingInAnyOrder("alias1"));
@ -1123,7 +1140,7 @@ public class IndicesAndAliasesResolverTests extends ESTestCase {
List<String> indices = resolveIndices(request, authorizedIndices).getLocal();
//the union of all resolved indices and aliases gets returned, including hidden indices as Get Aliases includes hidden by default
String[] expectedIndices = new String[]{"bar", "bar-closed", "foofoobar", "foobarfoo", "foofoo", "foofoo-closed", "hidden-open",
"hidden-closed", ".hidden-open", ".hidden-closed"};
"hidden-closed", ".hidden-open", ".hidden-closed", "date-hidden-" + todaySuffix, "date-hidden-" + tomorrowSuffix};
assertSameValues(indices, expectedIndices);
//_all gets replaced with all indices that user is authorized for
assertThat(request.indices(), arrayContainingInAnyOrder(expectedIndices));
@ -1139,12 +1156,14 @@ public class IndicesAndAliasesResolverTests extends ESTestCase {
List<String> indices = resolveIndices(request, authorizedIndices).getLocal();
//the union of all resolved indices and aliases gets returned, including hidden indices as Get Aliases includes hidden by default
String[] expectedIndices = new String[]{"bar", "bar-closed", "foofoobar", "foobarfoo", "foofoo", "foofoo-closed", "explicit",
"hidden-open", "hidden-closed", ".hidden-open", ".hidden-closed"};
"hidden-open", "hidden-closed", ".hidden-open", ".hidden-closed", "date-hidden-" + todaySuffix,
"date-hidden-" + tomorrowSuffix};
logger.info("indices: {}", indices);
assertSameValues(indices, expectedIndices);
//_all gets replaced with all indices that user is authorized for
assertThat(request.indices(), arrayContainingInAnyOrder("bar", "bar-closed", "foofoobar", "foobarfoo", "foofoo", "foofoo-closed",
"hidden-open", "hidden-closed", ".hidden-open", ".hidden-closed"));
"hidden-open", "hidden-closed", ".hidden-open", ".hidden-closed", "date-hidden-" + todaySuffix,
"date-hidden-" + tomorrowSuffix));
assertThat(request.aliases(), arrayContainingInAnyOrder("foofoobar", "foobarfoo", "explicit"));
}
@ -1157,7 +1176,7 @@ public class IndicesAndAliasesResolverTests extends ESTestCase {
List<String> indices = resolveIndices(request, authorizedIndices).getLocal();
//the union of all resolved indices and aliases gets returned, including hidden indices as Get Aliases includes hidden by default
String[] expectedIndices = new String[]{"bar", "bar-closed", "foofoobar", "foobarfoo", "foofoo", "foofoo-closed", "hidden-open",
"hidden-closed", ".hidden-open", ".hidden-closed"};
"hidden-closed", ".hidden-open", ".hidden-closed", "date-hidden-" + todaySuffix, "date-hidden-" + tomorrowSuffix};
assertSameValues(indices, expectedIndices);
//_all gets replaced with all indices that user is authorized for
assertThat(request.indices(), arrayContainingInAnyOrder(expectedIndices));
@ -1199,11 +1218,12 @@ public class IndicesAndAliasesResolverTests extends ESTestCase {
//the same and resolve those aliases to their corresponding concrete indices (which we let core do)
//also includes hidden indices as Get Aliases includes hidden by default
String[] expectedIndices = new String[]{"bar", "bar-closed", "foobarfoo", "foofoo", "foofoo-closed", "foofoobar", "hidden-open",
"hidden-closed", ".hidden-open", ".hidden-closed"};
"hidden-closed", ".hidden-open", ".hidden-closed", "date-hidden-" + todaySuffix, "date-hidden-" + tomorrowSuffix};
assertSameValues(indices, expectedIndices);
//alias foofoobar on both sides, that's fine, es core would do the same, same as above
assertThat(request.indices(), arrayContainingInAnyOrder("bar", "bar-closed", "foobarfoo", "foofoo", "foofoo-closed", "foofoobar",
"hidden-open", "hidden-closed", ".hidden-open", ".hidden-closed"));
"hidden-open", "hidden-closed", ".hidden-open", ".hidden-closed", "date-hidden-" + todaySuffix,
"date-hidden-" + tomorrowSuffix));
assertThat(request.aliases(), arrayContainingInAnyOrder("foofoobar"));
}
@ -1486,7 +1506,8 @@ public class IndicesAndAliasesResolverTests extends ESTestCase {
List<String> authorizedIndices = buildAuthorizedIndices(user, SearchAction.NAME);
ResolvedIndices resolvedIndices = defaultIndicesResolver.resolveIndicesAndAliases(searchRequest, metadata, authorizedIndices);
assertThat(resolvedIndices.getLocal(), containsInAnyOrder("bar", "bar-closed", "foofoobar", "foobarfoo", "foofoo", "foofoo-closed",
"hidden-open", "hidden-closed", ".hidden-open", ".hidden-closed"));
"hidden-open", "hidden-closed", ".hidden-open", ".hidden-closed", "date-hidden-" + todaySuffix,
"date-hidden-" + tomorrowSuffix));
assertThat(resolvedIndices.getRemote(), emptyIterable());
// open + hidden
@ -1494,11 +1515,12 @@ public class IndicesAndAliasesResolverTests extends ESTestCase {
searchRequest.indicesOptions(IndicesOptions.fromOptions(false, false, true, false, true));
resolvedIndices = defaultIndicesResolver.resolveIndicesAndAliases(searchRequest, metadata, authorizedIndices);
assertThat(resolvedIndices.getLocal(),
containsInAnyOrder("bar", "foofoobar", "foobarfoo", "foofoo", "hidden-open", ".hidden-open"));
containsInAnyOrder("bar", "foofoobar", "foobarfoo", "foofoo", "hidden-open", ".hidden-open", "date-hidden-" + todaySuffix,
"date-hidden-" + tomorrowSuffix));
assertThat(resolvedIndices.getRemote(), emptyIterable());
// open + implicit hidden for . indices
searchRequest = new SearchRequest(randomFrom(".*", ".hid*"));
searchRequest = new SearchRequest(randomFrom(".h*", ".hid*"));
searchRequest.indicesOptions(IndicesOptions.fromOptions(false, false, true, false, false));
authorizedIndices = buildAuthorizedIndices(user, SearchAction.NAME);
resolvedIndices = defaultIndicesResolver.resolveIndicesAndAliases(searchRequest, metadata, authorizedIndices);
@ -1514,7 +1536,7 @@ public class IndicesAndAliasesResolverTests extends ESTestCase {
assertThat(resolvedIndices.getRemote(), emptyIterable());
// closed + implicit hidden for . indices
searchRequest = new SearchRequest(randomFrom(".*", ".hid*"));
searchRequest = new SearchRequest(randomFrom(".h*", ".hid*"));
searchRequest.indicesOptions(IndicesOptions.fromOptions(false, false, false, true, false));
authorizedIndices = buildAuthorizedIndices(user, SearchAction.NAME);
resolvedIndices = defaultIndicesResolver.resolveIndicesAndAliases(searchRequest, metadata, authorizedIndices);
@ -1528,6 +1550,13 @@ public class IndicesAndAliasesResolverTests extends ESTestCase {
resolvedIndices = defaultIndicesResolver.resolveIndicesAndAliases(searchRequest, metadata, authorizedIndices);
assertThat(resolvedIndices.getLocal(), contains("-*"));
assertThat(resolvedIndices.getRemote(), emptyIterable());
// date math with default indices options
searchRequest = new SearchRequest("<date-hidden-{now/d}>");
authorizedIndices = buildAuthorizedIndices(user, SearchAction.NAME);
resolvedIndices = defaultIndicesResolver.resolveIndicesAndAliases(searchRequest, metadata, authorizedIndices);
assertThat(resolvedIndices.getLocal(), contains(oneOf("date-hidden-" + todaySuffix, "date-hidden-" + tomorrowSuffix)));
assertThat(resolvedIndices.getRemote(), emptyIterable());
}
public void testHiddenAliasesResolution() {