Added date math support in index names
Date math index name resolution enables you to search a range of time-series indices, rather than searching all of your time-series indices and filtering the the results or maintaining aliases. Limiting the number of indices that are searched reduces the load on the cluster and improves execution performance. For example, if you are searching for errors in your daily logs, you can use a date math name template to restrict the search to the past two days.
The added `ExpressionResolver` implementation that is responsible for resolving date math expressions in index names. This resolver is evaluated before wildcard expressions are evaluated.
The supported format: `<static_name{date_math_expr{date_format|timezone_id}}>` and the date math expressions must be enclosed within angle brackets. The `date_format` is optional and defaults to `YYYY.MM.dd`. The `timezone_id` id is optional too and defaults to `utc`.
The `{` character can be escaped by places `\\` before it.
Closes #12059
This commit is contained in:
Martijn van Groningen
parent
c423319dda
commit
ac3d090379
|
|
@ -155,6 +155,7 @@ import org.elasticsearch.action.suggest.SuggestAction;
|
|||
import org.elasticsearch.action.suggest.TransportSuggestAction;
|
||||
import org.elasticsearch.action.support.ActionFilter;
|
||||
import org.elasticsearch.action.support.ActionFilters;
|
||||
import org.elasticsearch.action.support.AutoCreateIndex;
|
||||
import org.elasticsearch.action.support.TransportAction;
|
||||
import org.elasticsearch.action.termvectors.*;
|
||||
import org.elasticsearch.action.termvectors.dfs.TransportDfsOnlyAction;
|
||||
|
|
@ -221,6 +222,7 @@ public class ActionModule extends AbstractModule {
|
|||
actionFilterMultibinder.addBinding().to(actionFilter);
|
||||
}
|
||||
bind(ActionFilters.class).asEagerSingleton();
|
||||
bind(AutoCreateIndex.class).asEagerSingleton();
|
||||
registerAction(NodesInfoAction.INSTANCE, TransportNodesInfoAction.class);
|
||||
registerAction(NodesStatsAction.INSTANCE, TransportNodesStatsAction.class);
|
||||
registerAction(NodesHotThreadsAction.INSTANCE, TransportNodesHotThreadsAction.class);
|
||||
|
|
|
|||
|
|
@ -74,13 +74,14 @@ public class TransportBulkAction extends HandledTransportAction<BulkRequest, Bul
|
|||
@Inject
|
||||
public TransportBulkAction(Settings settings, ThreadPool threadPool, TransportService transportService, ClusterService clusterService,
|
||||
TransportShardBulkAction shardBulkAction, TransportCreateIndexAction createIndexAction,
|
||||
ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) {
|
||||
ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver,
|
||||
AutoCreateIndex autoCreateIndex) {
|
||||
super(settings, BulkAction.NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver, BulkRequest.class);
|
||||
this.clusterService = clusterService;
|
||||
this.shardBulkAction = shardBulkAction;
|
||||
this.createIndexAction = createIndexAction;
|
||||
|
||||
this.autoCreateIndex = new AutoCreateIndex(settings);
|
||||
this.autoCreateIndex = autoCreateIndex;
|
||||
this.allowIdGeneration = this.settings.getAsBoolean("action.bulk.action.allow_id_generation", true);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -60,17 +60,19 @@ public class TransportDeleteAction extends TransportReplicationAction<DeleteRequ
|
|||
public TransportDeleteAction(Settings settings, TransportService transportService, ClusterService clusterService,
|
||||
IndicesService indicesService, ThreadPool threadPool, ShardStateAction shardStateAction,
|
||||
TransportCreateIndexAction createIndexAction, ActionFilters actionFilters,
|
||||
IndexNameExpressionResolver indexNameExpressionResolver, MappingUpdatedAction mappingUpdatedAction) {
|
||||
IndexNameExpressionResolver indexNameExpressionResolver, MappingUpdatedAction mappingUpdatedAction,
|
||||
AutoCreateIndex autoCreateIndex) {
|
||||
super(settings, DeleteAction.NAME, transportService, clusterService, indicesService, threadPool, shardStateAction,
|
||||
mappingUpdatedAction, actionFilters, indexNameExpressionResolver,
|
||||
DeleteRequest.class, DeleteRequest.class, ThreadPool.Names.INDEX);
|
||||
this.createIndexAction = createIndexAction;
|
||||
this.autoCreateIndex = new AutoCreateIndex(settings);
|
||||
this.autoCreateIndex = autoCreateIndex;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void doExecute(final DeleteRequest request, final ActionListener<DeleteResponse> listener) {
|
||||
if (autoCreateIndex.shouldAutoCreate(request.index(), clusterService.state())) {
|
||||
ClusterState state = clusterService.state();
|
||||
if (autoCreateIndex.shouldAutoCreate(request.index(), state)) {
|
||||
createIndexAction.execute(new CreateIndexRequest(request).index(request.index()).cause("auto(delete api)").masterNodeTimeout(request.timeout()), new ActionListener<CreateIndexResponse>() {
|
||||
@Override
|
||||
public void onResponse(CreateIndexResponse result) {
|
||||
|
|
|
|||
|
|
@ -75,11 +75,12 @@ public class TransportIndexAction extends TransportReplicationAction<IndexReques
|
|||
public TransportIndexAction(Settings settings, TransportService transportService, ClusterService clusterService,
|
||||
IndicesService indicesService, ThreadPool threadPool, ShardStateAction shardStateAction,
|
||||
TransportCreateIndexAction createIndexAction, MappingUpdatedAction mappingUpdatedAction,
|
||||
ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) {
|
||||
ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver,
|
||||
AutoCreateIndex autoCreateIndex) {
|
||||
super(settings, IndexAction.NAME, transportService, clusterService, indicesService, threadPool, shardStateAction, mappingUpdatedAction,
|
||||
actionFilters, indexNameExpressionResolver, IndexRequest.class, IndexRequest.class, ThreadPool.Names.INDEX);
|
||||
this.createIndexAction = createIndexAction;
|
||||
this.autoCreateIndex = new AutoCreateIndex(settings);
|
||||
this.autoCreateIndex = autoCreateIndex;
|
||||
this.allowIdGeneration = settings.getAsBoolean("action.allow_id_generation", true);
|
||||
this.clusterService = clusterService;
|
||||
}
|
||||
|
|
@ -87,7 +88,8 @@ public class TransportIndexAction extends TransportReplicationAction<IndexReques
|
|||
@Override
|
||||
protected void doExecute(final IndexRequest request, final ActionListener<IndexResponse> listener) {
|
||||
// if we don't have a master, we don't have metadata, that's fine, let it find a master using create index API
|
||||
if (autoCreateIndex.shouldAutoCreate(request.index(), clusterService.state())) {
|
||||
ClusterState state = clusterService.state();
|
||||
if (autoCreateIndex.shouldAutoCreate(request.index(), state)) {
|
||||
CreateIndexRequest createIndexRequest = new CreateIndexRequest(request);
|
||||
createIndexRequest.index(request.index());
|
||||
createIndexRequest.mapping(request.type());
|
||||
|
|
|
|||
|
|
@ -115,7 +115,10 @@ public abstract class TransportSearchTypeAction extends TransportAction<SearchRe
|
|||
|
||||
clusterState.blocks().globalBlockedRaiseException(ClusterBlockLevel.READ);
|
||||
|
||||
String[] concreteIndices = indexNameExpressionResolver.concreteIndices(clusterState, request.indicesOptions(), request.indices());
|
||||
// TODO: I think startTime() should become part of ActionRequest and that should be used both for index name
|
||||
// date math expressions and $now in scripts. This way all apis will deal with now in the same way instead
|
||||
// of just for the _search api
|
||||
String[] concreteIndices = indexNameExpressionResolver.concreteIndices(clusterState, request.indicesOptions(), startTime(), request.indices());
|
||||
|
||||
for (String index : concreteIndices) {
|
||||
clusterState.blocks().indexBlockedRaiseException(ClusterBlockLevel.READ, index);
|
||||
|
|
|
|||
|
|
@ -20,21 +20,28 @@
|
|||
package org.elasticsearch.action.support;
|
||||
|
||||
import org.elasticsearch.cluster.ClusterState;
|
||||
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
|
||||
import org.elasticsearch.common.Booleans;
|
||||
import org.elasticsearch.common.Strings;
|
||||
import org.elasticsearch.common.inject.Inject;
|
||||
import org.elasticsearch.common.regex.Regex;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
|
||||
/**
|
||||
* Encapsulates the logic of whether a new index should be automatically created when
|
||||
* a write operation is about to happen in a non existing index.
|
||||
*/
|
||||
public class AutoCreateIndex {
|
||||
public final class AutoCreateIndex {
|
||||
|
||||
private final boolean needToCheck;
|
||||
private final boolean globallyDisabled;
|
||||
private final String[] matches;
|
||||
private final String[] matches2;
|
||||
private final IndexNameExpressionResolver resolver;
|
||||
|
||||
public AutoCreateIndex(Settings settings) {
|
||||
@Inject
|
||||
public AutoCreateIndex(Settings settings, IndexNameExpressionResolver resolver) {
|
||||
this.resolver = resolver;
|
||||
String value = settings.get("action.auto_create_index");
|
||||
if (value == null || Booleans.isExplicitTrue(value)) {
|
||||
needToCheck = true;
|
||||
|
|
@ -71,7 +78,8 @@ public class AutoCreateIndex {
|
|||
if (!needToCheck) {
|
||||
return false;
|
||||
}
|
||||
if (state.metaData().hasConcreteIndex(index)) {
|
||||
boolean exists = resolver.hasIndexOrAlias(index, state);
|
||||
if (exists) {
|
||||
return false;
|
||||
}
|
||||
if (globallyDisabled) {
|
||||
|
|
|
|||
|
|
@ -75,14 +75,14 @@ public class TransportUpdateAction extends TransportInstanceSingleOperationActio
|
|||
public TransportUpdateAction(Settings settings, ThreadPool threadPool, ClusterService clusterService, TransportService transportService,
|
||||
TransportIndexAction indexAction, TransportDeleteAction deleteAction, TransportCreateIndexAction createIndexAction,
|
||||
UpdateHelper updateHelper, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver,
|
||||
IndicesService indicesService) {
|
||||
IndicesService indicesService, AutoCreateIndex autoCreateIndex) {
|
||||
super(settings, UpdateAction.NAME, threadPool, clusterService, transportService, actionFilters, indexNameExpressionResolver, UpdateRequest.class);
|
||||
this.indexAction = indexAction;
|
||||
this.deleteAction = deleteAction;
|
||||
this.createIndexAction = createIndexAction;
|
||||
this.updateHelper = updateHelper;
|
||||
this.indicesService = indicesService;
|
||||
this.autoCreateIndex = new AutoCreateIndex(settings);
|
||||
this.autoCreateIndex = autoCreateIndex;
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
|||
|
|
@ -21,31 +21,45 @@ package org.elasticsearch.cluster.metadata;
|
|||
|
||||
import com.google.common.base.Predicate;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import org.elasticsearch.ElasticsearchParseException;
|
||||
import org.elasticsearch.action.IndicesRequest;
|
||||
import org.elasticsearch.action.support.IndicesOptions;
|
||||
import org.elasticsearch.cluster.ClusterState;
|
||||
import org.elasticsearch.common.Nullable;
|
||||
import org.elasticsearch.common.Strings;
|
||||
import org.elasticsearch.common.collect.Tuple;
|
||||
import org.elasticsearch.common.component.AbstractComponent;
|
||||
import org.elasticsearch.common.inject.Inject;
|
||||
import org.elasticsearch.common.joda.DateMathParser;
|
||||
import org.elasticsearch.common.joda.FormatDateTimeFormatter;
|
||||
import org.elasticsearch.common.regex.Regex;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.index.Index;
|
||||
import org.elasticsearch.index.IndexNotFoundException;
|
||||
import org.elasticsearch.indices.IndexClosedException;
|
||||
import org.joda.time.DateTimeZone;
|
||||
import org.joda.time.format.DateTimeFormat;
|
||||
import org.joda.time.format.DateTimeFormatter;
|
||||
|
||||
import java.util.*;
|
||||
import java.util.concurrent.Callable;
|
||||
|
||||
import static com.google.common.collect.Lists.newArrayList;
|
||||
import static com.google.common.collect.Maps.filterEntries;
|
||||
import static com.google.common.collect.Maps.newHashMap;
|
||||
|
||||
public class IndexNameExpressionResolver {
|
||||
public class IndexNameExpressionResolver extends AbstractComponent {
|
||||
|
||||
private final ImmutableList<ExpressionResolver> expressionResolvers;
|
||||
private DateMathExpressionResolver dateMathExpressionResolver;
|
||||
|
||||
@Inject
|
||||
public IndexNameExpressionResolver() {
|
||||
expressionResolvers = ImmutableList.<ExpressionResolver>of(new WildcardExpressionResolver());
|
||||
public IndexNameExpressionResolver(Settings settings) {
|
||||
super(settings);
|
||||
expressionResolvers = ImmutableList.of(
|
||||
dateMathExpressionResolver = new DateMathExpressionResolver(settings),
|
||||
new WildcardExpressionResolver()
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -75,6 +89,24 @@ public class IndexNameExpressionResolver {
|
|||
return concreteIndices(context, indexExpressions);
|
||||
}
|
||||
|
||||
/**
|
||||
* Translates the provided index expression into actual concrete indices.
|
||||
*
|
||||
* @param state the cluster state containing all the data to resolve to expressions to concrete indices
|
||||
* @param options defines how the aliases or indices need to be resolved to concrete indices
|
||||
* @param startTime The start of the request where concrete indices is being invoked for
|
||||
* @param indexExpressions expressions that can be resolved to alias or index names.
|
||||
* @return the resolved concrete indices based on the cluster state, indices options and index expressions
|
||||
* provided indices options in the context don't allow such a case, or if the final result of the indices resolution
|
||||
* contains no indices and the indices options in the context don't allow such a case.
|
||||
* @throws IllegalArgumentException if one of the aliases resolve to multiple indices and the provided
|
||||
* indices options in the context don't allow such a case.
|
||||
*/
|
||||
public String[] concreteIndices(ClusterState state, IndicesOptions options, long startTime, String... indexExpressions) {
|
||||
Context context = new Context(state, options, startTime);
|
||||
return concreteIndices(context, indexExpressions);
|
||||
}
|
||||
|
||||
String[] concreteIndices(Context context, String... indexExpressions) {
|
||||
if (indexExpressions == null || indexExpressions.length == 0) {
|
||||
indexExpressions = new String[]{MetaData.ALL};
|
||||
|
|
@ -159,21 +191,30 @@ public class IndexNameExpressionResolver {
|
|||
* that require a single index as a result. The indices resolution must in fact return a single index when
|
||||
* using this method, an {@link IllegalArgumentException} gets thrown otherwise.
|
||||
*
|
||||
* @param request request containing the index or alias to be resolved to concrete index and
|
||||
* the indices options to be used for the index resolution
|
||||
* @throws IndexNotFoundException if the resolved index or alias provided doesn't exist
|
||||
* @param state the cluster state containing all the data to resolve to expression to a concrete index
|
||||
* @param request The request that defines how the an alias or an index need to be resolved to a concrete index
|
||||
* and the expression that can be resolved to an alias or an index name.
|
||||
* @throws IllegalArgumentException if the index resolution lead to more than one index
|
||||
* @return the concrete index obtained as a result of the index resolution
|
||||
*/
|
||||
public String concreteSingleIndex(ClusterState state, IndicesRequest request) {
|
||||
String indexOrAlias = request.indices() != null && request.indices().length > 0 ? request.indices()[0] : null;
|
||||
String[] indices = concreteIndices(state, request.indicesOptions(), indexOrAlias);
|
||||
String indexExpression = request.indices() != null && request.indices().length > 0 ? request.indices()[0] : null;
|
||||
String[] indices = concreteIndices(state, request.indicesOptions(), indexExpression);
|
||||
if (indices.length != 1) {
|
||||
throw new IllegalArgumentException("unable to return a single index as the index and options provided got resolved to multiple indices");
|
||||
}
|
||||
return indices[0];
|
||||
}
|
||||
|
||||
/**
|
||||
* @return whether the specified alias or index exists. If the alias or index contains datemath then that is resolved too.
|
||||
*/
|
||||
public boolean hasIndexOrAlias(String aliasOrIndex, ClusterState state) {
|
||||
Context context = new Context(state, IndicesOptions.lenientExpandOpen());
|
||||
String resolvedAliasOrIndex = dateMathExpressionResolver.resolveExpression(aliasOrIndex, context);
|
||||
return state.metaData().getAliasAndIndexLookup().containsKey(resolvedAliasOrIndex);
|
||||
}
|
||||
|
||||
/**
|
||||
* Iterates through the list of indices and selects the effective list of filtering aliases for the
|
||||
* given index.
|
||||
|
|
@ -405,10 +446,18 @@ public class IndexNameExpressionResolver {
|
|||
|
||||
private final ClusterState state;
|
||||
private final IndicesOptions options;
|
||||
private final long startTime;
|
||||
|
||||
Context(ClusterState state, IndicesOptions options) {
|
||||
this.state = state;
|
||||
this.options = options;
|
||||
startTime = System.currentTimeMillis();
|
||||
}
|
||||
|
||||
public Context(ClusterState state, IndicesOptions options, long startTime) {
|
||||
this.state = state;
|
||||
this.options = options;
|
||||
this.startTime = startTime;
|
||||
}
|
||||
|
||||
public ClusterState getState() {
|
||||
|
|
@ -418,6 +467,10 @@ public class IndexNameExpressionResolver {
|
|||
public IndicesOptions getOptions() {
|
||||
return options;
|
||||
}
|
||||
|
||||
public long getStartTime() {
|
||||
return startTime;
|
||||
}
|
||||
}
|
||||
|
||||
private interface ExpressionResolver {
|
||||
|
|
@ -582,4 +635,163 @@ public class IndexNameExpressionResolver {
|
|||
}
|
||||
}
|
||||
|
||||
final static class DateMathExpressionResolver implements ExpressionResolver {
|
||||
|
||||
private static final String EXPRESSION_LEFT_BOUND = "<";
|
||||
private static final String EXPRESSION_RIGHT_BOUND = ">";
|
||||
private static final char LEFT_BOUND = '{';
|
||||
private static final char RIGHT_BOUND = '}';
|
||||
private static final char ESCAPE_CHAR = '\\';
|
||||
private static final char TIME_ZONE_BOUND = '|';
|
||||
|
||||
private final DateTimeZone defaultTimeZone;
|
||||
private final String defaultDateFormatterPattern;
|
||||
private final DateTimeFormatter defaultDateFormatter;
|
||||
|
||||
public DateMathExpressionResolver(Settings settings) {
|
||||
String defaultTimeZoneId = settings.get("date_math_expression_resolver.default_time_zone", "UTC");
|
||||
this.defaultTimeZone = DateTimeZone.forID(defaultTimeZoneId);
|
||||
defaultDateFormatterPattern = settings.get("date_math_expression_resolver.default_date_format", "YYYY.MM.dd");
|
||||
this.defaultDateFormatter = DateTimeFormat.forPattern(defaultDateFormatterPattern);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<String> resolve(final Context context, List<String> expressions) {
|
||||
List<String> result = new ArrayList<>(expressions.size());
|
||||
for (String expression : expressions) {
|
||||
result.add(resolveExpression(expression, context));
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
String resolveExpression(String expression, final Context context) {
|
||||
if (expression.startsWith(EXPRESSION_LEFT_BOUND) == false || expression.endsWith(EXPRESSION_RIGHT_BOUND) == false) {
|
||||
return expression;
|
||||
}
|
||||
|
||||
boolean escape = false;
|
||||
boolean inDateFormat = false;
|
||||
boolean inPlaceHolder = false;
|
||||
final StringBuilder beforePlaceHolderSb = new StringBuilder();
|
||||
StringBuilder inPlaceHolderSb = new StringBuilder();
|
||||
final char[] text = expression.toCharArray();
|
||||
final int from = 1;
|
||||
final int length = text.length - 1;
|
||||
for (int i = from; i < length; i++) {
|
||||
boolean escapedChar = escape;
|
||||
if (escape) {
|
||||
escape = false;
|
||||
}
|
||||
|
||||
char c = text[i];
|
||||
if (c == ESCAPE_CHAR) {
|
||||
if (escapedChar) {
|
||||
beforePlaceHolderSb.append(c);
|
||||
escape = false;
|
||||
} else {
|
||||
escape = true;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (inPlaceHolder) {
|
||||
switch (c) {
|
||||
case LEFT_BOUND:
|
||||
if (inDateFormat && escapedChar) {
|
||||
inPlaceHolderSb.append(c);
|
||||
} else if (!inDateFormat) {
|
||||
inDateFormat = true;
|
||||
inPlaceHolderSb.append(c);
|
||||
} else {
|
||||
throw new ElasticsearchParseException("invalid dynamic name expression [{}]. invalid character in placeholder at position [{}]", new String(text, from, length), i);
|
||||
}
|
||||
break;
|
||||
|
||||
case RIGHT_BOUND:
|
||||
if (inDateFormat && escapedChar) {
|
||||
inPlaceHolderSb.append(c);
|
||||
} else if (inDateFormat) {
|
||||
inDateFormat = false;
|
||||
inPlaceHolderSb.append(c);
|
||||
} else {
|
||||
String inPlaceHolderString = inPlaceHolderSb.toString();
|
||||
int dateTimeFormatLeftBoundIndex = inPlaceHolderString.indexOf(LEFT_BOUND);
|
||||
String mathExpression;
|
||||
String dateFormatterPattern;
|
||||
DateTimeFormatter dateFormatter;
|
||||
final DateTimeZone timeZone;
|
||||
if (dateTimeFormatLeftBoundIndex < 0) {
|
||||
mathExpression = inPlaceHolderString;
|
||||
dateFormatterPattern = defaultDateFormatterPattern;
|
||||
dateFormatter = defaultDateFormatter;
|
||||
timeZone = defaultTimeZone;
|
||||
} else {
|
||||
if (inPlaceHolderString.lastIndexOf(RIGHT_BOUND) != inPlaceHolderString.length() - 1) {
|
||||
throw new ElasticsearchParseException("invalid dynamic name expression [{}]. missing closing `}` for date math format", inPlaceHolderString);
|
||||
}
|
||||
if (dateTimeFormatLeftBoundIndex == inPlaceHolderString.length() - 2) {
|
||||
throw new ElasticsearchParseException("invalid dynamic name expression [{}]. missing date format", inPlaceHolderString);
|
||||
}
|
||||
mathExpression = inPlaceHolderString.substring(0, dateTimeFormatLeftBoundIndex);
|
||||
String dateFormatterPatternAndTimeZoneId = inPlaceHolderString.substring(dateTimeFormatLeftBoundIndex + 1, inPlaceHolderString.length() - 1);
|
||||
int formatPatternTimeZoneSeparatorIndex = dateFormatterPatternAndTimeZoneId.indexOf(TIME_ZONE_BOUND);
|
||||
if (formatPatternTimeZoneSeparatorIndex != -1) {
|
||||
dateFormatterPattern = dateFormatterPatternAndTimeZoneId.substring(0, formatPatternTimeZoneSeparatorIndex);
|
||||
timeZone = DateTimeZone.forID(dateFormatterPatternAndTimeZoneId.substring(formatPatternTimeZoneSeparatorIndex + 1));
|
||||
} else {
|
||||
dateFormatterPattern = dateFormatterPatternAndTimeZoneId;
|
||||
timeZone = defaultTimeZone;
|
||||
}
|
||||
dateFormatter = DateTimeFormat.forPattern(dateFormatterPattern);
|
||||
}
|
||||
DateTimeFormatter parser = dateFormatter.withZone(timeZone);
|
||||
FormatDateTimeFormatter formatter = new FormatDateTimeFormatter(dateFormatterPattern, parser, Locale.ROOT);
|
||||
DateMathParser dateMathParser = new DateMathParser(formatter);
|
||||
long millis = dateMathParser.parse(mathExpression, new Callable<Long>() {
|
||||
@Override
|
||||
public Long call() throws Exception {
|
||||
return context.getStartTime();
|
||||
}
|
||||
}, false, timeZone);
|
||||
|
||||
String time = formatter.printer().print(millis);
|
||||
beforePlaceHolderSb.append(time);
|
||||
inPlaceHolderSb = new StringBuilder();
|
||||
inPlaceHolder = false;
|
||||
}
|
||||
break;
|
||||
|
||||
default:
|
||||
inPlaceHolderSb.append(c);
|
||||
}
|
||||
} else {
|
||||
switch (c) {
|
||||
case LEFT_BOUND:
|
||||
if (escapedChar) {
|
||||
beforePlaceHolderSb.append(c);
|
||||
} else {
|
||||
inPlaceHolder = true;
|
||||
}
|
||||
break;
|
||||
|
||||
case RIGHT_BOUND:
|
||||
if (!escapedChar) {
|
||||
throw new ElasticsearchParseException("invalid dynamic name expression [{}]. invalid character at position [{}]. " +
|
||||
"`{` and `}` are reserved characters and should be escaped when used as part of the index name using `\\` (e.g. `\\{text\\}`)", new String(text, from, length), i);
|
||||
}
|
||||
default:
|
||||
beforePlaceHolderSb.append(c);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (inPlaceHolder) {
|
||||
throw new ElasticsearchParseException("invalid dynamic name expression [{}]. date math placeholder is open ended", new String(text, from, length));
|
||||
}
|
||||
if (beforePlaceHolderSb.length() == 0) {
|
||||
throw new ElasticsearchParseException("nothing captured");
|
||||
}
|
||||
return beforePlaceHolderSb.toString();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -694,7 +694,7 @@ public class ShardReplicationTests extends ElasticsearchTestCase {
|
|||
ThreadPool threadPool) {
|
||||
super(settings, actionName, transportService, clusterService, null, threadPool,
|
||||
new ShardStateAction(settings, clusterService, transportService, null, null), null,
|
||||
new ActionFilters(new HashSet<ActionFilter>()), new IndexNameExpressionResolver(), Request.class, Request.class, ThreadPool.Names.SAME);
|
||||
new ActionFilters(new HashSet<ActionFilter>()), new IndexNameExpressionResolver(Settings.EMPTY), Request.class, Request.class, ThreadPool.Names.SAME);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
|||
|
|
@ -32,6 +32,7 @@ import org.elasticsearch.cluster.metadata.MetaData;
|
|||
import org.elasticsearch.cluster.routing.*;
|
||||
import org.elasticsearch.common.io.stream.BytesStreamOutput;
|
||||
import org.elasticsearch.common.io.stream.StreamInput;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.common.unit.TimeValue;
|
||||
import org.elasticsearch.index.shard.ShardId;
|
||||
import org.elasticsearch.test.ElasticsearchTestCase;
|
||||
|
|
@ -46,7 +47,7 @@ import static org.hamcrest.Matchers.*;
|
|||
|
||||
public class ClusterHealthResponsesTests extends ElasticsearchTestCase {
|
||||
|
||||
private final IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver();
|
||||
private final IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver(Settings.EMPTY);
|
||||
|
||||
private void assertIndexHealth(ClusterIndexHealth indexHealth, ShardCounter counter, IndexMetaData indexMetaData) {
|
||||
assertThat(indexHealth.getStatus(), equalTo(counter.status()));
|
||||
|
|
|
|||
|
|
@ -0,0 +1,199 @@
|
|||
/*
|
||||
* Licensed to Elasticsearch under one or more contributor
|
||||
* license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright
|
||||
* ownership. Elasticsearch licenses this file to you under
|
||||
* the Apache License, Version 2.0 (the "License"); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.cluster.metadata;
|
||||
|
||||
import org.elasticsearch.ElasticsearchParseException;
|
||||
import org.elasticsearch.action.support.IndicesOptions;
|
||||
import org.elasticsearch.cluster.ClusterName;
|
||||
import org.elasticsearch.cluster.ClusterState;
|
||||
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.Context;
|
||||
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.DateMathExpressionResolver;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.test.ElasticsearchTestCase;
|
||||
import org.joda.time.DateTime;
|
||||
import org.joda.time.DateTimeZone;
|
||||
import org.joda.time.format.DateTimeFormat;
|
||||
import org.junit.Test;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
|
||||
import static org.hamcrest.Matchers.equalTo;
|
||||
import static org.joda.time.DateTimeZone.UTC;
|
||||
|
||||
public class DateMathExpressionResolverTests extends ElasticsearchTestCase {
|
||||
|
||||
private final DateMathExpressionResolver expressionResolver = new DateMathExpressionResolver(Settings.EMPTY);
|
||||
private final Context context = new Context(
|
||||
ClusterState.builder(new ClusterName("_name")).build(), IndicesOptions.strictExpand()
|
||||
);
|
||||
|
||||
public void testNormal() throws Exception {
|
||||
int numIndexExpressions = randomIntBetween(1, 9);
|
||||
List<String> indexExpressions = new ArrayList<>(numIndexExpressions);
|
||||
for (int i = 0; i < numIndexExpressions; i++) {
|
||||
indexExpressions.add(randomAsciiOfLength(10));
|
||||
}
|
||||
List<String> result = expressionResolver.resolve(context, indexExpressions);
|
||||
assertThat(result.size(), equalTo(indexExpressions.size()));
|
||||
for (int i = 0; i < indexExpressions.size(); i++) {
|
||||
assertThat(result.get(i), equalTo(indexExpressions.get(i)));
|
||||
}
|
||||
}
|
||||
|
||||
public void testExpression() throws Exception {
|
||||
List<String> indexExpressions = Arrays.asList("<.marvel-{now}>", "<.watch_history-{now}>", "<logstash-{now}>");
|
||||
List<String> result = expressionResolver.resolve(context, indexExpressions);
|
||||
assertThat(result.size(), equalTo(3));
|
||||
assertThat(result.get(0), equalTo(".marvel-" + DateTimeFormat.forPattern("YYYY.MM.dd").print(new DateTime(context.getStartTime(), UTC))));
|
||||
assertThat(result.get(1), equalTo(".watch_history-" + DateTimeFormat.forPattern("YYYY.MM.dd").print(new DateTime(context.getStartTime(), UTC))));
|
||||
assertThat(result.get(2), equalTo("logstash-" + DateTimeFormat.forPattern("YYYY.MM.dd").print(new DateTime(context.getStartTime(), UTC))));
|
||||
}
|
||||
|
||||
public void testEmpty() throws Exception {
|
||||
List<String> result = expressionResolver.resolve(context, Collections.<String>emptyList());
|
||||
assertThat(result.size(), equalTo(0));
|
||||
}
|
||||
|
||||
public void testExpression_Static() throws Exception {
|
||||
List<String> result = expressionResolver.resolve(context, Arrays.asList("<.marvel-test>"));
|
||||
assertThat(result.size(), equalTo(1));
|
||||
assertThat(result.get(0), equalTo(".marvel-test"));
|
||||
}
|
||||
|
||||
public void testExpression_MultiParts() throws Exception {
|
||||
List<String> result = expressionResolver.resolve(context, Arrays.asList("<.text1-{now/d}-text2-{now/M}>"));
|
||||
assertThat(result.size(), equalTo(1));
|
||||
assertThat(result.get(0), equalTo(".text1-"
|
||||
+ DateTimeFormat.forPattern("YYYY.MM.dd").print(new DateTime(context.getStartTime(), UTC))
|
||||
+ "-text2-"
|
||||
+ DateTimeFormat.forPattern("YYYY.MM.dd").print(new DateTime(context.getStartTime(), UTC).withDayOfMonth(1))));
|
||||
}
|
||||
|
||||
public void testExpression_CustomFormat() throws Exception {
|
||||
List<String> results = expressionResolver.resolve(context, Arrays.asList("<.marvel-{now/d{YYYY.MM.dd}}>"));
|
||||
assertThat(results.size(), equalTo(1));
|
||||
assertThat(results.get(0), equalTo(".marvel-" + DateTimeFormat.forPattern("YYYY.MM.dd").print(new DateTime(context.getStartTime(), UTC))));
|
||||
}
|
||||
|
||||
public void testExpression_EscapeStatic() throws Exception {
|
||||
List<String> result = expressionResolver.resolve(context, Arrays.asList("<.mar\\{v\\}el-{now/d}>"));
|
||||
assertThat(result.size(), equalTo(1));
|
||||
assertThat(result.get(0), equalTo(".mar{v}el-" + DateTimeFormat.forPattern("YYYY.MM.dd").print(new DateTime(context.getStartTime(), UTC))));
|
||||
}
|
||||
|
||||
public void testExpression_EscapeDateFormat() throws Exception {
|
||||
List<String> result = expressionResolver.resolve(context, Arrays.asList("<.marvel-{now/d{'\\{year\\}'YYYY}}>"));
|
||||
assertThat(result.size(), equalTo(1));
|
||||
assertThat(result.get(0), equalTo(".marvel-" + DateTimeFormat.forPattern("'{year}'YYYY").print(new DateTime(context.getStartTime(), UTC))));
|
||||
}
|
||||
|
||||
public void testExpression_MixedArray() throws Exception {
|
||||
List<String> result = expressionResolver.resolve(context, Arrays.asList(
|
||||
"name1", "<.marvel-{now/d}>", "name2", "<.logstash-{now/M{YYYY.MM}}>"
|
||||
));
|
||||
assertThat(result.size(), equalTo(4));
|
||||
assertThat(result.get(0), equalTo("name1"));
|
||||
assertThat(result.get(1), equalTo(".marvel-" + DateTimeFormat.forPattern("YYYY.MM.dd").print(new DateTime(context.getStartTime(), UTC))));
|
||||
assertThat(result.get(2), equalTo("name2"));
|
||||
assertThat(result.get(3), equalTo(".logstash-" + DateTimeFormat.forPattern("YYYY.MM").print(new DateTime(context.getStartTime(), UTC).withDayOfMonth(1))));
|
||||
}
|
||||
|
||||
public void testExpression_CustomTimeZoneInSetting() throws Exception {
|
||||
DateTimeZone timeZone;
|
||||
int hoursOffset;
|
||||
int minutesOffset = 0;
|
||||
if (randomBoolean()) {
|
||||
hoursOffset = randomIntBetween(-12, 14);
|
||||
timeZone = DateTimeZone.forOffsetHours(hoursOffset);
|
||||
} else {
|
||||
hoursOffset = randomIntBetween(-11, 13);
|
||||
minutesOffset = randomIntBetween(0, 59);
|
||||
timeZone = DateTimeZone.forOffsetHoursMinutes(hoursOffset, minutesOffset);
|
||||
}
|
||||
DateTime now;
|
||||
if (hoursOffset >= 0) {
|
||||
// rounding to next day 00:00
|
||||
now = DateTime.now(UTC).plusHours(hoursOffset).plusMinutes(minutesOffset).withHourOfDay(0).withMinuteOfHour(0).withSecondOfMinute(0);
|
||||
} else {
|
||||
// rounding to today 00:00
|
||||
now = DateTime.now(UTC).withHourOfDay(0).withMinuteOfHour(0).withSecondOfMinute(0);
|
||||
}
|
||||
Settings settings = Settings.builder()
|
||||
.put("date_math_expression_resolver.default_time_zone", timeZone.getID())
|
||||
.build();
|
||||
DateMathExpressionResolver expressionResolver = new DateMathExpressionResolver(settings);
|
||||
Context context = new Context(this.context.getState(), this.context.getOptions(), now.getMillis());
|
||||
List<String> results = expressionResolver.resolve(context, Arrays.asList("<.marvel-{now/d{YYYY.MM.dd}}>"));
|
||||
assertThat(results.size(), equalTo(1));
|
||||
logger.info("timezone: [{}], now [{}], name: [{}]", timeZone, now, results.get(0));
|
||||
assertThat(results.get(0), equalTo(".marvel-" + DateTimeFormat.forPattern("YYYY.MM.dd").print(now.withZone(timeZone))));
|
||||
}
|
||||
|
||||
public void testExpression_CustomTimeZoneInIndexName() throws Exception {
|
||||
DateTimeZone timeZone;
|
||||
int hoursOffset;
|
||||
int minutesOffset = 0;
|
||||
if (randomBoolean()) {
|
||||
hoursOffset = randomIntBetween(-12, 14);
|
||||
timeZone = DateTimeZone.forOffsetHours(hoursOffset);
|
||||
} else {
|
||||
hoursOffset = randomIntBetween(-11, 13);
|
||||
minutesOffset = randomIntBetween(0, 59);
|
||||
timeZone = DateTimeZone.forOffsetHoursMinutes(hoursOffset, minutesOffset);
|
||||
}
|
||||
DateTime now;
|
||||
if (hoursOffset >= 0) {
|
||||
// rounding to next day 00:00
|
||||
now = DateTime.now(UTC).plusHours(hoursOffset).plusMinutes(minutesOffset).withHourOfDay(0).withMinuteOfHour(0).withSecondOfMinute(0);
|
||||
} else {
|
||||
// rounding to today 00:00
|
||||
now = DateTime.now(UTC).withHourOfDay(0).withMinuteOfHour(0).withSecondOfMinute(0);
|
||||
}
|
||||
Context context = new Context(this.context.getState(), this.context.getOptions(), now.getMillis());
|
||||
List<String> results = expressionResolver.resolve(context, Arrays.asList("<.marvel-{now/d{YYYY.MM.dd|" + timeZone.getID() + "}}>"));
|
||||
assertThat(results.size(), equalTo(1));
|
||||
logger.info("timezone: [{}], now [{}], name: [{}]", timeZone, now, results.get(0));
|
||||
assertThat(results.get(0), equalTo(".marvel-" + DateTimeFormat.forPattern("YYYY.MM.dd").print(now.withZone(timeZone))));
|
||||
}
|
||||
|
||||
@Test(expected = ElasticsearchParseException.class)
|
||||
public void testExpression_Invalid_Unescaped() throws Exception {
|
||||
expressionResolver.resolve(context, Arrays.asList("<.mar}vel-{now/d}>"));
|
||||
}
|
||||
|
||||
@Test(expected = ElasticsearchParseException.class)
|
||||
public void testExpression_Invalid_DateMathFormat() throws Exception {
|
||||
expressionResolver.resolve(context, Arrays.asList("<.marvel-{now/d{}>"));
|
||||
}
|
||||
|
||||
@Test(expected = ElasticsearchParseException.class)
|
||||
public void testExpression_Invalid_EmptyDateMathFormat() throws Exception {
|
||||
expressionResolver.resolve(context, Arrays.asList("<.marvel-{now/d{}}>"));
|
||||
}
|
||||
|
||||
@Test(expected = ElasticsearchParseException.class)
|
||||
public void testExpression_Invalid_OpenEnded() throws Exception {
|
||||
expressionResolver.resolve(context, Arrays.asList("<.marvel-{now/d>"));
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -27,6 +27,7 @@ import org.elasticsearch.cluster.ClusterState;
|
|||
import org.elasticsearch.cluster.metadata.IndexMetaData.State;
|
||||
import org.elasticsearch.common.Strings;
|
||||
import org.elasticsearch.index.IndexNotFoundException;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.indices.IndexClosedException;
|
||||
import org.elasticsearch.test.ElasticsearchTestCase;
|
||||
import org.junit.Test;
|
||||
|
|
@ -42,6 +43,8 @@ import static org.hamcrest.Matchers.*;
|
|||
*/
|
||||
public class IndexNameExpressionResolverTests extends ElasticsearchTestCase {
|
||||
|
||||
private final IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver(Settings.EMPTY);
|
||||
|
||||
@Test
|
||||
public void testIndexOptions_strict() {
|
||||
MetaData.Builder mdBuilder = MetaData.builder()
|
||||
|
|
@ -51,7 +54,6 @@ public class IndexNameExpressionResolverTests extends ElasticsearchTestCase {
|
|||
.put(indexBuilder("foofoo").putAlias(AliasMetaData.builder("barbaz")));
|
||||
|
||||
ClusterState state = ClusterState.builder(new ClusterName("_name")).metaData(mdBuilder).build();
|
||||
IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver();
|
||||
|
||||
IndicesOptions[] indicesOptions = new IndicesOptions[]{ IndicesOptions.strictExpandOpen(), IndicesOptions.strictExpand()};
|
||||
for (IndicesOptions options : indicesOptions) {
|
||||
|
|
@ -141,7 +143,6 @@ public class IndexNameExpressionResolverTests extends ElasticsearchTestCase {
|
|||
.put(indexBuilder("foofoo-closed").state(IndexMetaData.State.CLOSE))
|
||||
.put(indexBuilder("foofoo").putAlias(AliasMetaData.builder("barbaz")));
|
||||
ClusterState state = ClusterState.builder(new ClusterName("_name")).metaData(mdBuilder).build();
|
||||
IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver();
|
||||
|
||||
IndicesOptions lenientExpand = IndicesOptions.fromOptions(true, true, true, true);
|
||||
IndicesOptions[] indicesOptions = new IndicesOptions[]{ IndicesOptions.lenientExpandOpen(), lenientExpand};
|
||||
|
|
@ -210,7 +211,6 @@ public class IndexNameExpressionResolverTests extends ElasticsearchTestCase {
|
|||
.put(indexBuilder("foofoo-closed").state(IndexMetaData.State.CLOSE))
|
||||
.put(indexBuilder("foofoo").putAlias(AliasMetaData.builder("barbaz")));
|
||||
ClusterState state = ClusterState.builder(new ClusterName("_name")).metaData(mdBuilder).build();
|
||||
IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver();
|
||||
|
||||
IndicesOptions expandOpen = IndicesOptions.fromOptions(true, false, true, false);
|
||||
IndicesOptions expand = IndicesOptions.fromOptions(true, false, true, true);
|
||||
|
|
@ -260,7 +260,6 @@ public class IndexNameExpressionResolverTests extends ElasticsearchTestCase {
|
|||
.put(indexBuilder("bar"))
|
||||
.put(indexBuilder("foobar").putAlias(AliasMetaData.builder("barbaz")));
|
||||
ClusterState state = ClusterState.builder(new ClusterName("_name")).metaData(mdBuilder).build();
|
||||
IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver();
|
||||
|
||||
// Only closed
|
||||
IndicesOptions options = IndicesOptions.fromOptions(false, true, false, true);
|
||||
|
|
@ -333,7 +332,6 @@ public class IndexNameExpressionResolverTests extends ElasticsearchTestCase {
|
|||
.put(indexBuilder("foofoo-closed").state(IndexMetaData.State.CLOSE))
|
||||
.put(indexBuilder("foofoo").putAlias(AliasMetaData.builder("barbaz")));
|
||||
ClusterState state = ClusterState.builder(new ClusterName("_name")).metaData(mdBuilder).build();
|
||||
IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver();
|
||||
|
||||
//ignore unavailable and allow no indices
|
||||
{
|
||||
|
|
@ -428,7 +426,6 @@ public class IndexNameExpressionResolverTests extends ElasticsearchTestCase {
|
|||
.put(indexBuilder("foofoo-closed").state(IndexMetaData.State.CLOSE))
|
||||
.put(indexBuilder("foofoo").putAlias(AliasMetaData.builder("barbaz")));
|
||||
ClusterState state = ClusterState.builder(new ClusterName("_name")).metaData(mdBuilder).build();
|
||||
IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver();
|
||||
|
||||
//error on both unavailable and no indices + every alias needs to expand to a single index
|
||||
|
||||
|
|
@ -482,7 +479,6 @@ public class IndexNameExpressionResolverTests extends ElasticsearchTestCase {
|
|||
@Test
|
||||
public void testIndexOptions_emptyCluster() {
|
||||
ClusterState state = ClusterState.builder(new ClusterName("_name")).metaData(MetaData.builder().build()).build();
|
||||
IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver();
|
||||
|
||||
IndicesOptions options = IndicesOptions.strictExpandOpen();
|
||||
IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, options);
|
||||
|
|
@ -532,7 +528,6 @@ public class IndexNameExpressionResolverTests extends ElasticsearchTestCase {
|
|||
.put(indexBuilder("testXXX"))
|
||||
.put(indexBuilder("kuku"));
|
||||
ClusterState state = ClusterState.builder(new ClusterName("_name")).metaData(mdBuilder).build();
|
||||
IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver();
|
||||
IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, IndicesOptions.strictExpandOpen());
|
||||
|
||||
indexNameExpressionResolver.concreteIndices(context, "testZZZ");
|
||||
|
|
@ -544,7 +539,6 @@ public class IndexNameExpressionResolverTests extends ElasticsearchTestCase {
|
|||
.put(indexBuilder("testXXX"))
|
||||
.put(indexBuilder("kuku"));
|
||||
ClusterState state = ClusterState.builder(new ClusterName("_name")).metaData(mdBuilder).build();
|
||||
IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver();
|
||||
IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, IndicesOptions.lenientExpandOpen());
|
||||
|
||||
assertThat(newHashSet(indexNameExpressionResolver.concreteIndices(context, "testXXX", "testZZZ")), equalTo(newHashSet("testXXX")));
|
||||
|
|
@ -556,7 +550,6 @@ public class IndexNameExpressionResolverTests extends ElasticsearchTestCase {
|
|||
.put(indexBuilder("testXXX"))
|
||||
.put(indexBuilder("kuku"));
|
||||
ClusterState state = ClusterState.builder(new ClusterName("_name")).metaData(mdBuilder).build();
|
||||
IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver();
|
||||
IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, IndicesOptions.strictExpandOpen());
|
||||
|
||||
assertThat(newHashSet(indexNameExpressionResolver.concreteIndices(context, "testMo", "testMahdy")), equalTo(newHashSet("testXXX")));
|
||||
|
|
@ -568,9 +561,7 @@ public class IndexNameExpressionResolverTests extends ElasticsearchTestCase {
|
|||
.put(indexBuilder("testXXX"))
|
||||
.put(indexBuilder("kuku"));
|
||||
ClusterState state = ClusterState.builder(new ClusterName("_name")).metaData(mdBuilder).build();
|
||||
IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver();
|
||||
IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, IndicesOptions.lenientExpandOpen());
|
||||
|
||||
assertThat(newHashSet(indexNameExpressionResolver.concreteIndices(context, new String[]{})), equalTo(Sets.newHashSet("kuku", "testXXX")));
|
||||
}
|
||||
|
||||
|
|
@ -583,7 +574,6 @@ public class IndexNameExpressionResolverTests extends ElasticsearchTestCase {
|
|||
.put(indexBuilder("testYYY").state(State.OPEN))
|
||||
.put(indexBuilder("testYYX").state(State.OPEN));
|
||||
ClusterState state = ClusterState.builder(new ClusterName("_name")).metaData(mdBuilder).build();
|
||||
IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver();
|
||||
|
||||
IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, IndicesOptions.fromOptions(true, true, false, false));
|
||||
assertThat(newHashSet(indexNameExpressionResolver.concreteIndices(context, "testX*")), equalTo(new HashSet<String>()));
|
||||
|
|
@ -615,7 +605,6 @@ public class IndexNameExpressionResolverTests extends ElasticsearchTestCase {
|
|||
|
||||
IndicesOptions indicesOptions = IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean());
|
||||
ClusterState state = ClusterState.builder(new ClusterName("_name")).metaData(MetaData.builder().build()).build();
|
||||
IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver();
|
||||
IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, indicesOptions);
|
||||
|
||||
// with no indices, asking for all indices should return empty list or exception, depending on indices options
|
||||
|
|
@ -633,7 +622,6 @@ public class IndexNameExpressionResolverTests extends ElasticsearchTestCase {
|
|||
.put(indexBuilder("bbb").state(State.OPEN).putAlias(AliasMetaData.builder("bbb_alias1")))
|
||||
.put(indexBuilder("ccc").state(State.CLOSE).putAlias(AliasMetaData.builder("ccc_alias1")));
|
||||
state = ClusterState.builder(new ClusterName("_name")).metaData(mdBuilder).build();
|
||||
indexNameExpressionResolver = new IndexNameExpressionResolver();
|
||||
context = new IndexNameExpressionResolver.Context(state, indicesOptions);
|
||||
if (indicesOptions.expandWildcardsOpen() || indicesOptions.expandWildcardsClosed() || indicesOptions.allowNoIndices()) {
|
||||
String[] concreteIndices = indexNameExpressionResolver.concreteIndices(context, allIndices);
|
||||
|
|
@ -676,7 +664,6 @@ public class IndexNameExpressionResolverTests extends ElasticsearchTestCase {
|
|||
.put(indexBuilder("bbb").state(State.OPEN).putAlias(AliasMetaData.builder("bbb_alias1")))
|
||||
.put(indexBuilder("ccc").state(State.CLOSE).putAlias(AliasMetaData.builder("ccc_alias1")));
|
||||
ClusterState state = ClusterState.builder(new ClusterName("_name")).metaData(mdBuilder).build();
|
||||
IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver();
|
||||
IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, indicesOptions);
|
||||
|
||||
// asking for non existing wildcard pattern should return empty list or exception
|
||||
|
|
@ -760,7 +747,6 @@ public class IndexNameExpressionResolverTests extends ElasticsearchTestCase {
|
|||
//even though it does identify all indices, it's not a pattern but just an explicit list of them
|
||||
String[] concreteIndices = new String[]{"index1", "index2", "index3"};
|
||||
MetaData metaData = metaDataBuilder(concreteIndices);
|
||||
IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver();
|
||||
assertThat(indexNameExpressionResolver.isPatternMatchingAllIndices(metaData, concreteIndices, concreteIndices), equalTo(false));
|
||||
}
|
||||
|
||||
|
|
@ -769,7 +755,6 @@ public class IndexNameExpressionResolverTests extends ElasticsearchTestCase {
|
|||
String[] indicesOrAliases = new String[]{"*"};
|
||||
String[] concreteIndices = new String[]{"index1", "index2", "index3"};
|
||||
MetaData metaData = metaDataBuilder(concreteIndices);
|
||||
IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver();
|
||||
assertThat(indexNameExpressionResolver.isPatternMatchingAllIndices(metaData, indicesOrAliases, concreteIndices), equalTo(true));
|
||||
}
|
||||
|
||||
|
|
@ -778,7 +763,6 @@ public class IndexNameExpressionResolverTests extends ElasticsearchTestCase {
|
|||
String[] indicesOrAliases = new String[]{"index*"};
|
||||
String[] concreteIndices = new String[]{"index1", "index2", "index3"};
|
||||
MetaData metaData = metaDataBuilder(concreteIndices);
|
||||
IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver();
|
||||
assertThat(indexNameExpressionResolver.isPatternMatchingAllIndices(metaData, indicesOrAliases, concreteIndices), equalTo(true));
|
||||
}
|
||||
|
||||
|
|
@ -788,7 +772,6 @@ public class IndexNameExpressionResolverTests extends ElasticsearchTestCase {
|
|||
String[] concreteIndices = new String[]{"index1", "index2", "index3"};
|
||||
String[] allConcreteIndices = new String[]{"index1", "index2", "index3", "a", "b"};
|
||||
MetaData metaData = metaDataBuilder(allConcreteIndices);
|
||||
IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver();
|
||||
assertThat(indexNameExpressionResolver.isPatternMatchingAllIndices(metaData, indicesOrAliases, concreteIndices), equalTo(false));
|
||||
}
|
||||
|
||||
|
|
@ -797,7 +780,6 @@ public class IndexNameExpressionResolverTests extends ElasticsearchTestCase {
|
|||
String[] indicesOrAliases = new String[]{"-index1", "+index1"};
|
||||
String[] concreteIndices = new String[]{"index1", "index2", "index3"};
|
||||
MetaData metaData = metaDataBuilder(concreteIndices);
|
||||
IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver();
|
||||
assertThat(indexNameExpressionResolver.isPatternMatchingAllIndices(metaData, indicesOrAliases, concreteIndices), equalTo(true));
|
||||
}
|
||||
|
||||
|
|
@ -807,7 +789,6 @@ public class IndexNameExpressionResolverTests extends ElasticsearchTestCase {
|
|||
String[] concreteIndices = new String[]{"index2", "index3"};
|
||||
String[] allConcreteIndices = new String[]{"index1", "index2", "index3"};
|
||||
MetaData metaData = metaDataBuilder(allConcreteIndices);
|
||||
IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver();
|
||||
assertThat(indexNameExpressionResolver.isPatternMatchingAllIndices(metaData, indicesOrAliases, concreteIndices), equalTo(false));
|
||||
}
|
||||
|
||||
|
|
@ -816,7 +797,6 @@ public class IndexNameExpressionResolverTests extends ElasticsearchTestCase {
|
|||
String[] indicesOrAliases = new String[]{"index*", "-index1", "+index1"};
|
||||
String[] concreteIndices = new String[]{"index1", "index2", "index3"};
|
||||
MetaData metaData = metaDataBuilder(concreteIndices);
|
||||
IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver();
|
||||
assertThat(indexNameExpressionResolver.isPatternMatchingAllIndices(metaData, indicesOrAliases, concreteIndices), equalTo(true));
|
||||
}
|
||||
|
||||
|
|
@ -826,7 +806,6 @@ public class IndexNameExpressionResolverTests extends ElasticsearchTestCase {
|
|||
String[] concreteIndices = new String[]{"index2", "index3"};
|
||||
String[] allConcreteIndices = new String[]{"index1", "index2", "index3"};
|
||||
MetaData metaData = metaDataBuilder(allConcreteIndices);
|
||||
IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver();
|
||||
assertThat(indexNameExpressionResolver.isPatternMatchingAllIndices(metaData, indicesOrAliases, concreteIndices), equalTo(false));
|
||||
}
|
||||
|
||||
|
|
@ -837,7 +816,6 @@ public class IndexNameExpressionResolverTests extends ElasticsearchTestCase {
|
|||
.put(indexBuilder("foo2-closed").state(IndexMetaData.State.CLOSE).putAlias(AliasMetaData.builder("foobar2-closed")))
|
||||
.put(indexBuilder("foo3").putAlias(AliasMetaData.builder("foobar2-closed")));
|
||||
ClusterState state = ClusterState.builder(new ClusterName("_name")).metaData(mdBuilder).build();
|
||||
IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver();
|
||||
|
||||
IndexNameExpressionResolver.Context context = new IndexNameExpressionResolver.Context(state, IndicesOptions.strictExpandOpenAndForbidClosed());
|
||||
try {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,87 @@
|
|||
/*
|
||||
* Licensed to Elasticsearch under one or more contributor
|
||||
* license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright
|
||||
* ownership. Elasticsearch licenses this file to you under
|
||||
* the Apache License, Version 2.0 (the "License"); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.indices;
|
||||
|
||||
import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
|
||||
import org.elasticsearch.action.delete.DeleteResponse;
|
||||
import org.elasticsearch.action.get.GetResponse;
|
||||
import org.elasticsearch.action.search.SearchResponse;
|
||||
import org.elasticsearch.test.ElasticsearchIntegrationTest;
|
||||
import org.elasticsearch.test.hamcrest.ElasticsearchAssertions;
|
||||
import org.joda.time.DateTime;
|
||||
import org.joda.time.DateTimeZone;
|
||||
import org.joda.time.format.DateTimeFormat;
|
||||
|
||||
import static org.hamcrest.Matchers.equalTo;
|
||||
import static org.hamcrest.Matchers.is;
|
||||
import static org.hamcrest.Matchers.notNullValue;
|
||||
|
||||
public class DateMathIndexExpressionsIntegrationTests extends ElasticsearchIntegrationTest {
|
||||
|
||||
public void testIndexNameDateMathExpressions() {
|
||||
DateTime now = new DateTime(DateTimeZone.UTC);
|
||||
String index1 = ".marvel-" + DateTimeFormat.forPattern("YYYY.MM.dd").print(now);
|
||||
String index2 = ".marvel-" + DateTimeFormat.forPattern("YYYY.MM.dd").print(now.minusDays(1));
|
||||
String index3 = ".marvel-" + DateTimeFormat.forPattern("YYYY.MM.dd").print(now.minusDays(2));
|
||||
createIndex(index1, index2, index3);
|
||||
|
||||
String dateMathExp1 = "<.marvel-{now/d}>";
|
||||
String dateMathExp2 = "<.marvel-{now/d-1d}>";
|
||||
String dateMathExp3 = "<.marvel-{now/d-2d}>";
|
||||
client().prepareIndex(dateMathExp1, "type", "1").setSource("{}").get();
|
||||
client().prepareIndex(dateMathExp2, "type", "2").setSource("{}").get();
|
||||
client().prepareIndex(dateMathExp3, "type", "3").setSource("{}").get();
|
||||
refresh();
|
||||
|
||||
SearchResponse searchResponse = client().prepareSearch(dateMathExp1, dateMathExp2, dateMathExp3).get();
|
||||
ElasticsearchAssertions.assertHitCount(searchResponse, 3);
|
||||
ElasticsearchAssertions.assertSearchHits(searchResponse, "1", "2", "3");
|
||||
|
||||
GetResponse getResponse = client().prepareGet(dateMathExp1, "type", "1").get();
|
||||
assertThat(getResponse.isExists(), is(true));
|
||||
assertThat(getResponse.getId(), equalTo("1"));
|
||||
|
||||
getResponse = client().prepareGet(dateMathExp2, "type", "2").get();
|
||||
assertThat(getResponse.isExists(), is(true));
|
||||
assertThat(getResponse.getId(), equalTo("2"));
|
||||
|
||||
getResponse = client().prepareGet(dateMathExp3, "type", "3").get();
|
||||
assertThat(getResponse.isExists(), is(true));
|
||||
assertThat(getResponse.getId(), equalTo("3"));
|
||||
|
||||
IndicesStatsResponse indicesStatsResponse = client().admin().indices().prepareStats(dateMathExp1, dateMathExp2, dateMathExp3).get();
|
||||
assertThat(indicesStatsResponse.getIndex(index1), notNullValue());
|
||||
assertThat(indicesStatsResponse.getIndex(index2), notNullValue());
|
||||
assertThat(indicesStatsResponse.getIndex(index3), notNullValue());
|
||||
|
||||
DeleteResponse deleteResponse = client().prepareDelete(dateMathExp1, "type", "1").get();
|
||||
assertThat(deleteResponse.isFound(), equalTo(true));
|
||||
assertThat(deleteResponse.getId(), equalTo("1"));
|
||||
|
||||
deleteResponse = client().prepareDelete(dateMathExp2, "type", "2").get();
|
||||
assertThat(deleteResponse.isFound(), equalTo(true));
|
||||
assertThat(deleteResponse.getId(), equalTo("2"));
|
||||
|
||||
deleteResponse = client().prepareDelete(dateMathExp3, "type", "3").get();
|
||||
assertThat(deleteResponse.isFound(), equalTo(true));
|
||||
assertThat(deleteResponse.getId(), equalTo("3"));
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -9,6 +9,7 @@ The conventions listed in this chapter can be applied throughout the REST
|
|||
API, unless otherwise specified.
|
||||
|
||||
* <<multi-index>>
|
||||
* <<date-math-index-names>>
|
||||
* <<common-options>>
|
||||
|
||||
--
|
||||
|
|
@ -55,6 +56,76 @@ The defaults settings for the above parameters depend on the api being used.
|
|||
NOTE: Single index APIs such as the <<docs>> and the
|
||||
<<indices-aliases,single-index `alias` APIs>> do not support multiple indices.
|
||||
|
||||
[[date-math-index-names]]
|
||||
== Date math support in index names
|
||||
|
||||
Date math index name resolution enables you to search a range of time-series indices, rather
|
||||
than searching all of your time-series indices and filtering the results or maintaining aliases.
|
||||
Limiting the number of indices that are searched reduces the load on the cluster and improves
|
||||
execution performance. For example, if you are searching for errors in your
|
||||
daily logs, you can use a date math name template to restrict the search to the past
|
||||
two days.
|
||||
|
||||
Almost all APIs that have an `index` parameter, support date math in the `index` parameter
|
||||
value.
|
||||
|
||||
A date math index name takes the following form:
|
||||
|
||||
[source,txt]
|
||||
----------------------------------------------------------------------
|
||||
<static_name{date_math_expr{date_format|time_zone}}>
|
||||
----------------------------------------------------------------------
|
||||
|
||||
Where:
|
||||
|
||||
[horizontal]
|
||||
`static_name`:: is the static text part of the name
|
||||
`date_math_expr`:: is a dynamic date math expression that computes the date dynamically
|
||||
`date_format`:: is the optional format in which the computed date should be rendered. Defaults to `YYYY.MM.dd`.
|
||||
`time_zone`:: is the optional time zone . Defaults to `utc`.
|
||||
|
||||
You must enclose date math index name expressions within angle brackets. For example:
|
||||
|
||||
[source,js]
|
||||
----------------------------------------------------------------------
|
||||
curl -XGET 'localhost:9200/<logstash-{now/d-2d}>/_search' {
|
||||
"query" : {
|
||||
...
|
||||
}
|
||||
}
|
||||
----------------------------------------------------------------------
|
||||
|
||||
The following example shows different forms of date math index names and the final index names
|
||||
they resolve to given the current time is 22rd March 2024 noon utc.
|
||||
|
||||
[options="header"]
|
||||
|======
|
||||
| Expression |Resolves to
|
||||
| `<logstash-{now/d}>` | `logstash-2024.03.22`
|
||||
| `<logstash-{now/M}>` | `logstash-2024.03.01`
|
||||
| `<logstash-{now/M{YYYY.MM}}>` | `logstash-2024.03`
|
||||
| `<logstash-{now/M-1M{YYYY.MM}}>` | `logstash-2024.02`
|
||||
| `<logstash-{now/d{YYYY.MM.dd\|+12:00}}` | `logstash-2024.03.23`
|
||||
|======
|
||||
|
||||
To use the characters `{` and `}` in the static part of an index name template, escape them
|
||||
with a backslash `\`, for example:
|
||||
|
||||
* `<elastic\\{ON\\}-{now/M}>` resolves to `elastic{ON}-2024.03.01`
|
||||
|
||||
The following example shows a search request that searches the Logstash indices for the past
|
||||
three days, assuming the indices use the default Logstash index name format,
|
||||
`logstash-YYYY.MM.dd`.
|
||||
|
||||
[source,js]
|
||||
----------------------------------------------------------------------
|
||||
curl -XGET 'localhost:9200/<logstash-{now/d-2d}>,<logstash-{now/d-1d}>,<logstash-{now/d}>/_search' {
|
||||
"query" : {
|
||||
...
|
||||
}
|
||||
}
|
||||
----------------------------------------------------------------------
|
||||
|
||||
[[common-options]]
|
||||
== Common options
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue