honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into feature/query-refactoring 

core/src/test/java/org/elasticsearch/aliases/IndexAliasesTests.java
This commit is contained in:
Christoph Büscher 2015-07-27 14:11:51 +02:00
parent 3d53413650 20facccaf6
commit 0128252075
29 changed files with 779 additions and 495 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
core/src/main/java/org/elasticsearch/cluster/ClusterChangedEvent.java
View File

@ -110,8 +110,11 @@ public class ClusterChangedEvent {
// is actually supposed to be deleted or imported as dangling instead. for example a new master might not have
// the index in its cluster state because it was started with an empty data folder and in this case we want to
// import as dangling. we check here for new master too to be on the safe side in this case.
// norelease because we are not sure this is actually a good solution
// See discussion on https://github.com/elastic/elasticsearch/pull/9952
// This means that under certain conditions deleted indices might be reimported if a master fails while the deletion
// request is issued and a node receives the cluster state that would trigger the deletion from the new master.
// See test MetaDataWriteDataNodesTests.testIndicesDeleted()
// See discussion on https://github.com/elastic/elasticsearch/pull/9952 and
// https://github.com/elastic/elasticsearch/issues/11665
if (hasNewMaster() || previousState == null) {
return ImmutableList.of();
}

138
core/src/main/java/org/elasticsearch/cluster/metadata/AliasOrIndex.java Normal file
View File

@ -0,0 +1,138 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.cluster.metadata;
import com.google.common.collect.UnmodifiableIterator;
import org.elasticsearch.common.collect.Tuple;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
/**
* Encapsulates the {@link IndexMetaData} instances of a concrete index or indices an alias is pointing to.
*/
public interface AliasOrIndex {
/**
* @return whether this an alias or concrete index
*/
boolean isAlias();
/**
* @return All {@link IndexMetaData} of all concrete indices this alias is referring to or if this is a concrete index its {@link IndexMetaData}
*/
List<IndexMetaData> getIndices();
/**
* Represents an concrete index and encapsulates its {@link IndexMetaData}
*/
class Index implements AliasOrIndex {
private final IndexMetaData concreteIndex;
public Index(IndexMetaData indexMetaData) {
this.concreteIndex = indexMetaData;
}
@Override
public boolean isAlias() {
return false;
}
@Override
public List<IndexMetaData> getIndices() {
return Collections.singletonList(concreteIndex);
}
/**
* @return If this is an concrete index, its {@link IndexMetaData}
*/
public IndexMetaData getIndex() {
return concreteIndex;
}
}
/**
* Represents an alias and groups all {@link IndexMetaData} instances sharing the same alias name together.
*/
class Alias implements AliasOrIndex {
private final String aliasName;
private final List<IndexMetaData> referenceIndexMetaDatas;
public Alias(AliasMetaData aliasMetaData, IndexMetaData indexMetaData) {
this.aliasName = aliasMetaData.getAlias();
this.referenceIndexMetaDatas = new ArrayList<>();
this.referenceIndexMetaDatas.add(indexMetaData);
}
@Override
public boolean isAlias() {
return true;
}
@Override
public List<IndexMetaData> getIndices() {
return referenceIndexMetaDatas;
}
/**
* Returns the unique alias metadata per concrete index.
*
* (note that although alias can point to the same concrete indices, each alias reference may have its own routing
* and filters)
*/
public Iterable<Tuple<String, AliasMetaData>> getConcreteIndexAndAliasMetaDatas() {
return new Iterable<Tuple<String, AliasMetaData>>() {
@Override
public Iterator<Tuple<String, AliasMetaData>> iterator() {
return new UnmodifiableIterator<Tuple<String,AliasMetaData>>() {
int index = 0;
@Override
public boolean hasNext() {
return index < referenceIndexMetaDatas.size();
}
@Override
public Tuple<String, AliasMetaData> next() {
IndexMetaData indexMetaData = referenceIndexMetaDatas.get(index++);
return new Tuple<>(indexMetaData.getIndex(), indexMetaData.getAliases().get(aliasName));
}
};
}
};
}
public AliasMetaData getFirstAliasMetaData() {
return referenceIndexMetaDatas.get(0).getAliases().get(aliasName);
}
void addIndex(IndexMetaData indexMetaData) {
this.referenceIndexMetaDatas.add(indexMetaData);
}
}
}

243
core/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java
View File

@ -19,16 +19,14 @@
package org.elasticsearch.cluster.metadata;
import com.carrotsearch.hppc.cursors.ObjectCursor;
import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
import com.google.common.base.Predicate;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import org.elasticsearch.action.IndicesRequest;
import org.elasticsearch.action.support.IndicesOptions;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.regex.Regex;
import org.elasticsearch.index.Index;
@ -38,6 +36,7 @@ import org.elasticsearch.indices.IndexClosedException;
import java.util.*;
import static com.google.common.collect.Lists.newArrayList;
import static com.google.common.collect.Maps.filterEntries;
import static com.google.common.collect.Maps.newHashMap;
public class IndexNameExpressionResolver {
@ -108,44 +107,40 @@ public class IndexNameExpressionResolver {
List<String> concreteIndices = new ArrayList<>(expressions.size());
for (String expression : expressions) {
List<IndexMetaData> indexMetaDatas;
IndexMetaData indexMetaData = metaData.getIndices().get(expression);
if (indexMetaData == null) {
ImmutableOpenMap<String, AliasMetaData> indexAliasMap = metaData.aliases().get(expression);
if (indexAliasMap == null) {
if (failNoIndices) {
IndexNotFoundException infe = new IndexNotFoundException(expression);
infe.setResources("index_expression", expression);
throw infe;
} else {
continue;
}
AliasOrIndex aliasOrIndex = metaData.getAliasAndIndexLookup().get(expression);
if (aliasOrIndex == null) {
if (failNoIndices) {
IndexNotFoundException infe = new IndexNotFoundException(expression);
infe.setResources("index_expression", expression);
throw infe;
} else {
continue;
}
if (indexAliasMap.size() > 1 && !options.allowAliasesToMultipleIndices()) {
throw new IllegalArgumentException("Alias [" + expression + "] has more than one indices associated with it [" + Arrays.toString(indexAliasMap.keys().toArray(String.class)) + "], can't execute a single index op");
}
indexMetaDatas = new ArrayList<>(indexAliasMap.size());
for (ObjectObjectCursor<String, AliasMetaData> cursor : indexAliasMap) {
indexMetaDatas.add(metaData.getIndices().get(cursor.key));
}
} else {
indexMetaDatas = Collections.singletonList(indexMetaData);
}
for (IndexMetaData found : indexMetaDatas) {
if (found.getState() == IndexMetaData.State.CLOSE) {
Collection<IndexMetaData> resolvedIndices = aliasOrIndex.getIndices();
if (resolvedIndices.size() > 1 && !options.allowAliasesToMultipleIndices()) {
String[] indexNames = new String[resolvedIndices.size()];
int i = 0;
for (IndexMetaData indexMetaData : resolvedIndices) {
indexNames[i++] = indexMetaData.getIndex();
}
throw new IllegalArgumentException("Alias [" + expression + "] has more than one indices associated with it [" + Arrays.toString(indexNames) + "], can't execute a single index op");
}
for (IndexMetaData index : resolvedIndices) {
if (index.getState() == IndexMetaData.State.CLOSE) {
if (failClosed) {
throw new IndexClosedException(new Index(found.getIndex()));
throw new IndexClosedException(new Index(index.getIndex()));
} else {
if (options.forbidClosedIndices() == false) {
concreteIndices.add(found.getIndex());
concreteIndices.add(index.getIndex());
}
}
} else if (found.getState() == IndexMetaData.State.OPEN) {
concreteIndices.add(found.getIndex());
} else if (index.getState() == IndexMetaData.State.OPEN) {
concreteIndices.add(index.getIndex());
} else {
throw new IllegalStateException("index state [" + found.getState() + "] not supported");
throw new IllegalStateException("index state [" + index.getState() + "] not supported");
}
}
}
@ -264,10 +259,6 @@ public class IndexNameExpressionResolver {
return resolveSearchRoutingAllIndices(state.metaData(), routing);
}
if (resolvedExpressions.size() == 1) {
return resolveSearchRoutingSingleValue(state.metaData(), routing, resolvedExpressions.get(0));
}
Map<String, Set<String>> routings = null;
Set<String> paramRouting = null;
// List of indices that don't require any routing
@ -277,40 +268,43 @@ public class IndexNameExpressionResolver {
}
for (String expression : resolvedExpressions) {
ImmutableOpenMap<String, AliasMetaData> indexToRoutingMap = state.metaData().getAliases().get(expression);
if (indexToRoutingMap != null && !indexToRoutingMap.isEmpty()) {
for (ObjectObjectCursor<String, AliasMetaData> indexRouting : indexToRoutingMap) {
if (!norouting.contains(indexRouting.key)) {
if (!indexRouting.value.searchRoutingValues().isEmpty()) {
AliasOrIndex aliasOrIndex = state.metaData().getAliasAndIndexLookup().get(expression);
if (aliasOrIndex != null && aliasOrIndex.isAlias()) {
AliasOrIndex.Alias alias = (AliasOrIndex.Alias) aliasOrIndex;
for (Tuple<String, AliasMetaData> item : alias.getConcreteIndexAndAliasMetaDatas()) {
String concreteIndex = item.v1();
AliasMetaData aliasMetaData = item.v2();
if (!norouting.contains(concreteIndex)) {
if (!aliasMetaData.searchRoutingValues().isEmpty()) {
// Routing alias
if (routings == null) {
routings = newHashMap();
}
Set<String> r = routings.get(indexRouting.key);
Set<String> r = routings.get(concreteIndex);
if (r == null) {
r = new HashSet<>();
routings.put(indexRouting.key, r);
routings.put(concreteIndex, r);
}
r.addAll(indexRouting.value.searchRoutingValues());
r.addAll(aliasMetaData.searchRoutingValues());
if (paramRouting != null) {
r.retainAll(paramRouting);
}
if (r.isEmpty()) {
routings.remove(indexRouting.key);
routings.remove(concreteIndex);
}
} else {
// Non-routing alias
if (!norouting.contains(indexRouting.key)) {
norouting.add(indexRouting.key);
if (!norouting.contains(concreteIndex)) {
norouting.add(concreteIndex);
if (paramRouting != null) {
Set<String> r = new HashSet<>(paramRouting);
if (routings == null) {
routings = newHashMap();
}
routings.put(indexRouting.key, r);
routings.put(concreteIndex, r);
} else {
if (routings != null) {
routings.remove(indexRouting.key);
routings.remove(concreteIndex);
}
}
}
@ -342,49 +336,6 @@ public class IndexNameExpressionResolver {
return routings;
}
private Map<String, Set<String>> resolveSearchRoutingSingleValue(MetaData metaData, @Nullable String routing, String aliasOrIndex) {
Map<String, Set<String>> routings = null;
Set<String> paramRouting = null;
if (routing != null) {
paramRouting = Strings.splitStringByCommaToSet(routing);
}
ImmutableOpenMap<String, AliasMetaData> indexToRoutingMap = metaData.getAliases().get(aliasOrIndex);
if (indexToRoutingMap != null && !indexToRoutingMap.isEmpty()) {
// It's an alias
for (ObjectObjectCursor<String, AliasMetaData> indexRouting : indexToRoutingMap) {
if (!indexRouting.value.searchRoutingValues().isEmpty()) {
// Routing alias
Set<String> r = new HashSet<>(indexRouting.value.searchRoutingValues());
if (paramRouting != null) {
r.retainAll(paramRouting);
}
if (!r.isEmpty()) {
if (routings == null) {
routings = newHashMap();
}
routings.put(indexRouting.key, r);
}
} else {
// Non-routing alias
if (paramRouting != null) {
Set<String> r = new HashSet<>(paramRouting);
if (routings == null) {
routings = newHashMap();
}
routings.put(indexRouting.key, r);
}
}
}
} else {
// It's an index
if (paramRouting != null) {
routings = ImmutableMap.of(aliasOrIndex, paramRouting);
}
}
return routings;
}
/**
* Sets the same routing for all indices
*/
@ -494,7 +445,7 @@ public class IndexNameExpressionResolver {
return expressions;
}
if (expressions.isEmpty() || (expressions.size() == 1 && MetaData.ALL.equals(expressions.get(0)))) {
if (expressions.isEmpty() || (expressions.size() == 1 && (MetaData.ALL.equals(expressions.get(0))) || Regex.isMatchAllPattern(expressions.get(0)))) {
if (options.expandWildcardsOpen() && options.expandWildcardsClosed()) {
return Arrays.asList(metaData.concreteAllIndices());
} else if (options.expandWildcardsOpen()) {
@ -508,22 +459,22 @@ public class IndexNameExpressionResolver {
Set<String> result = null;
for (int i = 0; i < expressions.size(); i++) {
String aliasOrIndex = expressions.get(i);
if (metaData.getAliasAndIndexMap().containsKey(aliasOrIndex)) {
String expression = expressions.get(i);
if (metaData.getAliasAndIndexLookup().containsKey(expression)) {
if (result != null) {
result.add(aliasOrIndex);
result.add(expression);
}
continue;
}
boolean add = true;
if (aliasOrIndex.charAt(0) == '+') {
if (expression.charAt(0) == '+') {
// if its the first, add empty result set
if (i == 0) {
result = new HashSet<>();
}
add = true;
aliasOrIndex = aliasOrIndex.substring(1);
} else if (aliasOrIndex.charAt(0) == '-') {
expression = expression.substring(1);
} else if (expression.charAt(0) == '-') {
// if its the first, fill it with all the indices...
if (i == 0) {
String[] concreteIndices;
@ -540,19 +491,19 @@ public class IndexNameExpressionResolver {
result = new HashSet<>(Arrays.asList(concreteIndices));
}
add = false;
aliasOrIndex = aliasOrIndex.substring(1);
expression = expression.substring(1);
}
if (!Regex.isSimpleMatchPattern(aliasOrIndex)) {
if (!options.ignoreUnavailable() && !metaData.getAliasAndIndexMap().containsKey(aliasOrIndex)) {
IndexNotFoundException infe = new IndexNotFoundException(aliasOrIndex);
infe.setResources("index_or_alias", aliasOrIndex);
if (!Regex.isSimpleMatchPattern(expression)) {
if (!options.ignoreUnavailable() && !metaData.getAliasAndIndexLookup().containsKey(expression)) {
IndexNotFoundException infe = new IndexNotFoundException(expression);
infe.setResources("index_or_alias", expression);
throw infe;
}
if (result != null) {
if (add) {
result.add(aliasOrIndex);
result.add(expression);
} else {
result.remove(aliasOrIndex);
result.remove(expression);
}
}
continue;
@ -562,44 +513,60 @@ public class IndexNameExpressionResolver {
result = new HashSet<>();
result.addAll(expressions.subList(0, i));
}
String[] indices;
if (options.expandWildcardsOpen() && options.expandWildcardsClosed()) {
indices = metaData.concreteAllIndices();
} else if (options.expandWildcardsOpen()) {
indices = metaData.concreteAllOpenIndices();
} else if (options.expandWildcardsClosed()) {
indices = metaData.concreteAllClosedIndices();
final IndexMetaData.State excludeState;
if (options.expandWildcardsOpen() && options.expandWildcardsClosed()){
excludeState = null;
} else if (options.expandWildcardsOpen() && options.expandWildcardsClosed() == false) {
excludeState = IndexMetaData.State.CLOSE;
} else if (options.expandWildcardsClosed() && options.expandWildcardsOpen() == false) {
excludeState = IndexMetaData.State.OPEN;
} else {
assert false : "this shouldn't get called if wildcards expand to none";
indices = Strings.EMPTY_ARRAY;
excludeState = null;
}
boolean found = false;
// iterating over all concrete indices and see if there is a wildcard match
for (String index : indices) {
if (Regex.simpleMatch(aliasOrIndex, index)) {
found = true;
if (add) {
result.add(index);
} else {
result.remove(index);
final Map<String, AliasOrIndex> matches;
if (Regex.isMatchAllPattern(expression)) {
// Can only happen if the expressions was initially: '-*'
matches = metaData.getAliasAndIndexLookup();
} else if (expression.endsWith("*")) {
// Suffix wildcard:
assert expression.length() >= 2 : "expression [" + expression + "] should have at least a length of 2";
String fromPrefix = expression.substring(0, expression.length() - 1);
char[] toPrefixCharArr = fromPrefix.toCharArray();
toPrefixCharArr[toPrefixCharArr.length - 1]++;
String toPrefix = new String(toPrefixCharArr);
matches = metaData.getAliasAndIndexLookup().subMap(fromPrefix, toPrefix);
} else {
// Other wildcard expressions:
final String pattern = expression;
matches = filterEntries(metaData.getAliasAndIndexLookup(), new Predicate<Map.Entry<String, AliasOrIndex>>() {
@Override
public boolean apply(@Nullable Map.Entry<String, AliasOrIndex> input) {
return Regex.simpleMatch(pattern, input.getKey());
}
});
}
for (Map.Entry<String, AliasOrIndex> entry : matches.entrySet()) {
AliasOrIndex aliasOrIndex = entry.getValue();
if (aliasOrIndex.isAlias() == false) {
AliasOrIndex.Index index = (AliasOrIndex.Index) aliasOrIndex;
if (excludeState != null && index.getIndex().getState() == excludeState) {
continue;
}
}
}
// iterating over all aliases and see if there is a wildcard match
for (ObjectCursor<String> cursor : metaData.getAliases().keys()) {
String alias = cursor.value;
if (Regex.simpleMatch(aliasOrIndex, alias)) {
found = true;
if (add) {
result.add(alias);
} else {
result.remove(alias);
}
if (add) {
result.add(entry.getKey());
} else {
result.remove(entry.getKey());
}
}
if (!found && !options.allowNoIndices()) {
IndexNotFoundException infe = new IndexNotFoundException(aliasOrIndex);
infe.setResources("index_or_alias", aliasOrIndex);
if (matches.isEmpty() && options.allowNoIndices() == false) {
IndexNotFoundException infe = new IndexNotFoundException(expression);
infe.setResources("index_or_alias", expression);
throw infe;
}
}

204
core/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java
View File

@ -19,7 +19,6 @@
package org.elasticsearch.cluster.metadata;
import com.carrotsearch.hppc.ObjectArrayList;
import com.carrotsearch.hppc.ObjectHashSet;
import com.carrotsearch.hppc.cursors.ObjectCursor;
import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
@ -48,7 +47,6 @@ import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.settings.loader.SettingsLoader;
import org.elasticsearch.common.xcontent.*;
import org.elasticsearch.discovery.DiscoverySettings;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.IndexNotFoundException;
import org.elasticsearch.indices.recovery.RecoverySettings;
import org.elasticsearch.indices.store.IndicesStore;
@ -146,16 +144,14 @@ public class MetaData implements Iterable<IndexMetaData>, Diffable<MetaData>, Fr
private final transient int totalNumberOfShards; // Transient ? not serializable anyway?
private final int numberOfShards;
private final String[] allIndices;
private final String[] allOpenIndices;
private final String[] allClosedIndices;
private final ImmutableOpenMap<String, ImmutableOpenMap<String, AliasMetaData>> aliases;
private final ImmutableOpenMap<String, String[]> aliasAndIndexToIndexMap;
private final SortedMap<String, AliasOrIndex> aliasAndIndexLookup;
@SuppressWarnings("unchecked")
MetaData(String clusterUUID, long version, Settings transientSettings, Settings persistentSettings, ImmutableOpenMap<String, IndexMetaData> indices, ImmutableOpenMap<String, IndexTemplateMetaData> templates, ImmutableOpenMap<String, Custom> customs) {
MetaData(String clusterUUID, long version, Settings transientSettings, Settings persistentSettings, ImmutableOpenMap<String, IndexMetaData> indices, ImmutableOpenMap<String, IndexTemplateMetaData> templates, ImmutableOpenMap<String, Custom> customs, String[] allIndices, String[] allOpenIndices, String[] allClosedIndices, SortedMap<String, AliasOrIndex> aliasAndIndexLookup) {
this.clusterUUID = clusterUUID;
this.version = version;
this.transientSettings = transientSettings;
@ -166,87 +162,17 @@ public class MetaData implements Iterable<IndexMetaData>, Diffable<MetaData>, Fr
this.templates = templates;
int totalNumberOfShards = 0;
int numberOfShards = 0;
int numAliases = 0;
for (ObjectCursor<IndexMetaData> cursor : indices.values()) {
totalNumberOfShards += cursor.value.totalNumberOfShards();
numberOfShards += cursor.value.numberOfShards();
numAliases += cursor.value.aliases().size();
}
this.totalNumberOfShards = totalNumberOfShards;
this.numberOfShards = numberOfShards;
// build all indices map
List<String> allIndicesLst = Lists.newArrayList();
for (ObjectCursor<IndexMetaData> cursor : indices.values()) {
allIndicesLst.add(cursor.value.index());
}
allIndices = allIndicesLst.toArray(new String[allIndicesLst.size()]);
int numIndices = allIndicesLst.size();
List<String> allOpenIndices = Lists.newArrayList();
List<String> allClosedIndices = Lists.newArrayList();
for (ObjectCursor<IndexMetaData> cursor : indices.values()) {
IndexMetaData indexMetaData = cursor.value;
if (indexMetaData.state() == IndexMetaData.State.OPEN) {
allOpenIndices.add(indexMetaData.index());
} else if (indexMetaData.state() == IndexMetaData.State.CLOSE) {
allClosedIndices.add(indexMetaData.index());
}
}
this.allOpenIndices = allOpenIndices.toArray(new String[allOpenIndices.size()]);
this.allClosedIndices = allClosedIndices.toArray(new String[allClosedIndices.size()]);
// build aliases map
ImmutableOpenMap.Builder<String, Object> tmpAliases = ImmutableOpenMap.builder(numAliases);
for (ObjectCursor<IndexMetaData> cursor : indices.values()) {
IndexMetaData indexMetaData = cursor.value;
String index = indexMetaData.index();
for (ObjectCursor<AliasMetaData> aliasCursor : indexMetaData.aliases().values()) {
AliasMetaData aliasMd = aliasCursor.value;
ImmutableOpenMap.Builder<String, AliasMetaData> indexAliasMap = (ImmutableOpenMap.Builder<String, AliasMetaData>) tmpAliases.get(aliasMd.alias());
if (indexAliasMap == null) {
indexAliasMap = ImmutableOpenMap.builder(1); // typically, there is 1 alias pointing to an index
tmpAliases.put(aliasMd.alias(), indexAliasMap);
}
indexAliasMap.put(index, aliasMd);
}
}
for (ObjectCursor<String> cursor : tmpAliases.keys()) {
String alias = cursor.value;
// if there is access to the raw values buffer of the map that the immutable maps wraps, then we don't need to use put, and just set array slots
ImmutableOpenMap<String, AliasMetaData> map = ((ImmutableOpenMap.Builder) tmpAliases.get(alias)).cast().build();
tmpAliases.put(alias, map);
}
this.aliases = tmpAliases.<String, ImmutableOpenMap<String, AliasMetaData>>cast().build();
ImmutableOpenMap.Builder<String, Object> aliasAndIndexToIndexMap = ImmutableOpenMap.builder(numAliases + numIndices);
for (ObjectCursor<IndexMetaData> cursor : indices.values()) {
IndexMetaData indexMetaData = cursor.value;
ObjectArrayList<String> indicesLst = (ObjectArrayList<String>) aliasAndIndexToIndexMap.get(indexMetaData.index());
if (indicesLst == null) {
indicesLst = new ObjectArrayList<>();
aliasAndIndexToIndexMap.put(indexMetaData.index(), indicesLst);
}
indicesLst.add(indexMetaData.index());
for (ObjectCursor<String> cursor1 : indexMetaData.aliases().keys()) {
String alias = cursor1.value;
indicesLst = (ObjectArrayList<String>) aliasAndIndexToIndexMap.get(alias);
if (indicesLst == null) {
indicesLst = new ObjectArrayList<>();
aliasAndIndexToIndexMap.put(alias, indicesLst);
}
indicesLst.add(indexMetaData.index());
}
}
for (ObjectObjectCursor<String, Object> cursor : aliasAndIndexToIndexMap) {
String[] indicesLst = ((ObjectArrayList<String>) cursor.value).toArray(String.class);
aliasAndIndexToIndexMap.put(cursor.key, indicesLst);
}
this.aliasAndIndexToIndexMap = aliasAndIndexToIndexMap.<String, String[]>cast().build();
this.allIndices = allIndices;
this.allOpenIndices = allOpenIndices;
this.allClosedIndices = allClosedIndices;
this.aliasAndIndexLookup = aliasAndIndexLookup;
}
public long version() {
@ -272,16 +198,32 @@ public class MetaData implements Iterable<IndexMetaData>, Diffable<MetaData>, Fr
return this.persistentSettings;
}
public ImmutableOpenMap<String, ImmutableOpenMap<String, AliasMetaData>> aliases() {
return this.aliases;
public boolean hasAlias(String alias) {
AliasOrIndex aliasOrIndex = getAliasAndIndexLookup().get(alias);
if (aliasOrIndex != null) {
return aliasOrIndex.isAlias();
} else {
return false;
}
}
public ImmutableOpenMap<String, ImmutableOpenMap<String, AliasMetaData>> getAliases() {
return aliases();
public boolean equalsAliases(MetaData other) {
for (ObjectCursor<IndexMetaData> cursor : other.indices().values()) {
IndexMetaData otherIndex = cursor.value;
IndexMetaData thisIndex= indices().get(otherIndex.getIndex());
if (thisIndex == null) {
return false;
}
if (otherIndex.getAliases().equals(thisIndex.getAliases()) == false) {
return false;
}
}
return true;
}
public ImmutableOpenMap<String, String[]> getAliasAndIndexMap() {
return aliasAndIndexToIndexMap;
public SortedMap<String, AliasOrIndex> getAliasAndIndexLookup() {
return aliasAndIndexLookup;
}
/**
@ -477,15 +419,24 @@ public class MetaData implements Iterable<IndexMetaData>, Diffable<MetaData>, Fr
// TODO: This can be moved to IndexNameExpressionResolver too, but this means that we will support wildcards and other expressions
// in the index,bulk,update and delete apis.
public String resolveIndexRouting(@Nullable String routing, String aliasOrIndex) {
// Check if index is specified by an alias
ImmutableOpenMap<String, AliasMetaData> indexAliases = aliases.get(aliasOrIndex);
if (indexAliases == null || indexAliases.isEmpty()) {
if (aliasOrIndex == null) {
return routing;
}
if (indexAliases.size() > 1) {
throw new IllegalArgumentException("Alias [" + aliasOrIndex + "] has more than one index associated with it [" + Arrays.toString(indexAliases.keys().toArray(String.class)) + "], can't execute a single index op");
AliasOrIndex result = getAliasAndIndexLookup().get(aliasOrIndex);
if (result == null || result.isAlias() == false) {
return routing;
}
AliasMetaData aliasMd = indexAliases.values().iterator().next().value;
AliasOrIndex.Alias alias = (AliasOrIndex.Alias) result;
if (result.getIndices().size() > 1) {
String[] indexNames = new String[result.getIndices().size()];
int i = 0;
for (IndexMetaData indexMetaData : result.getIndices()) {
indexNames[i++] = indexMetaData.getIndex();
}
throw new IllegalArgumentException("Alias [" + aliasOrIndex + "] has more than one index associated with it [" + Arrays.toString(indexNames) + "], can't execute a single index op");
}
AliasMetaData aliasMd = alias.getFirstAliasMetaData();
if (aliasMd.indexRouting() != null) {
if (routing != null) {
if (!routing.equals(aliasMd.indexRouting())) {
@ -507,7 +458,7 @@ public class MetaData implements Iterable<IndexMetaData>, Diffable<MetaData>, Fr
}
public boolean hasConcreteIndex(String index) {
return aliasAndIndexToIndexMap.containsKey(index);
return getAliasAndIndexLookup().containsKey(index);
}
public IndexMetaData index(String index) {
@ -829,13 +780,18 @@ public class MetaData implements Iterable<IndexMetaData>, Diffable<MetaData>, Fr
}
if (newPersistentSettings != null) {
return new MetaData(metaData.clusterUUID(),
metaData.version(),
metaData.transientSettings(),
newPersistentSettings.build(),
metaData.getIndices(),
metaData.getTemplates(),
metaData.getCustoms());
return new MetaData(
metaData.clusterUUID(),
metaData.version(),
metaData.transientSettings(),
newPersistentSettings.build(),
metaData.getIndices(),
metaData.getTemplates(),
metaData.getCustoms(),
metaData.concreteAllIndices(),
metaData.concreteAllOpenIndices(),
metaData.concreteAllClosedIndices(),
metaData.getAliasAndIndexLookup());
} else {
// No changes:
return metaData;
@ -1013,7 +969,53 @@ public class MetaData implements Iterable<IndexMetaData>, Diffable<MetaData>, Fr
}
public MetaData build() {
return new MetaData(clusterUUID, version, transientSettings, persistentSettings, indices.build(), templates.build(), customs.build());
// TODO: We should move these datastructures to IndexNameExpressionResolver, this will give the following benefits:
// 1) The datastructures will only be rebuilded when needed. Now during serailizing we rebuild these datastructures
// while these datastructures aren't even used.
// 2) The aliasAndIndexLookup can be updated instead of rebuilding it all the time.
// build all concrete indices arrays:
// TODO: I think we can remove these arrays. it isn't worth the effort, for operations on all indices.
// When doing an operation across all indices, most of the time is spent on actually going to all shards and
// do the required operations, the bottleneck isn't resolving expressions into concrete indices.
List<String> allIndicesLst = Lists.newArrayList();
for (ObjectCursor<IndexMetaData> cursor : indices.values()) {
allIndicesLst.add(cursor.value.index());
}
String[] allIndices = allIndicesLst.toArray(new String[allIndicesLst.size()]);
List<String> allOpenIndicesLst = Lists.newArrayList();
List<String> allClosedIndicesLst = Lists.newArrayList();
for (ObjectCursor<IndexMetaData> cursor : indices.values()) {
IndexMetaData indexMetaData = cursor.value;
if (indexMetaData.state() == IndexMetaData.State.OPEN) {
allOpenIndicesLst.add(indexMetaData.index());
} else if (indexMetaData.state() == IndexMetaData.State.CLOSE) {
allClosedIndicesLst.add(indexMetaData.index());
}
}
String[] allOpenIndices = allOpenIndicesLst.toArray(new String[allOpenIndicesLst.size()]);
String[] allClosedIndices = allClosedIndicesLst.toArray(new String[allClosedIndicesLst.size()]);
// build all indices map
SortedMap<String, AliasOrIndex> aliasAndIndexLookup = new TreeMap<>();
for (ObjectCursor<IndexMetaData> cursor : indices.values()) {
IndexMetaData indexMetaData = cursor.value;
aliasAndIndexLookup.put(indexMetaData.getIndex(), new AliasOrIndex.Index(indexMetaData));
for (ObjectObjectCursor<String, AliasMetaData> aliasCursor : indexMetaData.getAliases()) {
AliasMetaData aliasMetaData = aliasCursor.value;
AliasOrIndex.Alias aliasOrIndex = (AliasOrIndex.Alias) aliasAndIndexLookup.get(aliasMetaData.getAlias());
if (aliasOrIndex == null) {
aliasOrIndex = new AliasOrIndex.Alias(aliasMetaData, indexMetaData);
aliasAndIndexLookup.put(aliasMetaData.getAlias(), aliasOrIndex);
} else {
aliasOrIndex.addIndex(indexMetaData);
}
}
}
aliasAndIndexLookup = Collections.unmodifiableSortedMap(aliasAndIndexLookup);
return new MetaData(clusterUUID, version, transientSettings, persistentSettings, indices.build(), templates.build(), customs.build(), allIndices, allOpenIndices, allClosedIndices, aliasAndIndexLookup);
}
public static String toXContent(MetaData metaData) throws IOException {

2
core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java
View File

@ -181,7 +181,7 @@ public class MetaDataCreateIndexService extends AbstractComponent {
"index name is too long, (" + byteCount +
" > " + MAX_INDEX_NAME_BYTES + ")");
}
if (state.metaData().aliases().containsKey(index)) {
if (state.metaData().hasAlias(index)) {
throw new InvalidIndexNameException(new Index(index), index, "already exists as alias");
}
}

2
core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexAliasesService.java
View File

@ -145,7 +145,7 @@ public class MetaDataIndexAliasesService extends AbstractComponent {
ClusterState updatedState = ClusterState.builder(currentState).metaData(builder).build();
// even though changes happened, they resulted in 0 actual changes to metadata
// i.e. remove and add the same alias to the same index
if (!updatedState.metaData().aliases().equals(currentState.metaData().aliases())) {
if (!updatedState.metaData().equalsAliases(currentState.metaData())) {
return updatedState;
}
}

22
core/src/main/java/org/elasticsearch/common/settings/loader/XContentSettingsLoader.java
View File

@ -20,6 +20,7 @@
package org.elasticsearch.common.settings.loader;
import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.common.xcontent.XContent;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentType;
@ -65,6 +66,27 @@ public abstract class XContentSettingsLoader implements SettingsLoader {
throw new ElasticsearchParseException("malformed, expected settings to start with 'object', instead was [{}]", token);
}
serializeObject(settings, sb, path, jp, null);
// ensure we reached the end of the stream
XContentParser.Token lastToken = null;
try {
while (!jp.isClosed() && (lastToken = jp.nextToken()) == null);
} catch (Exception e) {
throw new ElasticsearchParseException(
"malformed, expected end of settings but encountered additional content starting at line number: [{}], column number: [{}]",
e,
jp.getTokenLocation().lineNumber,
jp.getTokenLocation().columnNumber
);
}
if (lastToken != null) {
throw new ElasticsearchParseException(
"malformed, expected end of settings but encountered additional content starting at line number: [{}], column number: [{}]",
jp.getTokenLocation().lineNumber,
jp.getTokenLocation().columnNumber
);
}
return settings;
}

2
core/src/main/java/org/elasticsearch/common/xcontent/XContentParser.java
View File

@ -250,4 +250,6 @@ public interface XContentParser extends Releasable {
* @return last token's location or null if cannot be determined
*/
XContentLocation getTokenLocation();
boolean isClosed();
}

5
core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentParser.java
View File

@ -248,4 +248,9 @@ public class JsonXContentParser extends AbstractXContentParser {
}
throw new IllegalStateException("No matching token for json_token [" + token + "]");
}
@Override
public boolean isClosed() {
return parser.isClosed();
}
}

3
core/src/main/java/org/elasticsearch/common/xcontent/support/AbstractXContentParser.java
View File

@ -319,4 +319,7 @@ public abstract class AbstractXContentParser implements XContentParser {
}
return null;
}
@Override
public abstract boolean isClosed();
}

21
core/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java
View File

@ -28,7 +28,8 @@ import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.metadata.MetaData;
import org.elasticsearch.cluster.metadata.MetaDataIndexUpgradeService;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.cluster.routing.*;
import org.elasticsearch.cluster.routing.RoutingNode;
import org.elasticsearch.cluster.routing.ShardRouting;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.component.AbstractComponent;
import org.elasticsearch.common.inject.Inject;
@ -98,6 +99,7 @@ public class GatewayMetaState extends AbstractComponent implements ClusterStateL
@Override
public void clusterChanged(ClusterChangedEvent event) {
Set<String> relevantIndices = new HashSet<>();
final ClusterState state = event.state();
if (state.blocks().disableStatePersistence()) {
@ -148,7 +150,7 @@ public class GatewayMetaState extends AbstractComponent implements ClusterStateL
}
Iterable<IndexMetaWriteInfo> writeInfo;
relevantIndices = getRelevantIndices(event.state(), previouslyWrittenIndices);
relevantIndices = getRelevantIndices(event.state(), event.previousState(), previouslyWrittenIndices);
writeInfo = resolveStatesToBeWritten(previouslyWrittenIndices, relevantIndices, previousMetaData, event.state().metaData());
// check and write changes in indices
for (IndexMetaWriteInfo indexMetaWrite : writeInfo) {
@ -169,10 +171,10 @@ public class GatewayMetaState extends AbstractComponent implements ClusterStateL
}
}
public static Set<String> getRelevantIndices(ClusterState state, ImmutableSet<String> previouslyWrittenIndices) {
public static Set<String> getRelevantIndices(ClusterState state, ClusterState previousState,ImmutableSet<String> previouslyWrittenIndices) {
Set<String> relevantIndices;
if (isDataOnlyNode(state)) {
relevantIndices = getRelevantIndicesOnDataOnlyNode(state, previouslyWrittenIndices);
relevantIndices = getRelevantIndicesOnDataOnlyNode(state, previousState, previouslyWrittenIndices);
} else if (state.nodes().localNode().masterNode() == true) {
relevantIndices = getRelevantIndicesForMasterEligibleNode(state);
} else {
@ -278,7 +280,7 @@ public class GatewayMetaState extends AbstractComponent implements ClusterStateL
return indicesToWrite;
}
public static Set<String> getRelevantIndicesOnDataOnlyNode(ClusterState state, ImmutableSet<String> previouslyWrittenIndices) {
public static Set<String> getRelevantIndicesOnDataOnlyNode(ClusterState state, ClusterState previousState, ImmutableSet<String> previouslyWrittenIndices) {
RoutingNode newRoutingNode = state.getRoutingNodes().node(state.nodes().localNodeId());
if (newRoutingNode == null) {
throw new IllegalStateException("cluster state does not contain this node - cannot write index meta state");
@ -289,7 +291,14 @@ public class GatewayMetaState extends AbstractComponent implements ClusterStateL
}
// we have to check the meta data also: closed indices will not appear in the routing table, but we must still write the state if we have it written on disk previously
for (IndexMetaData indexMetaData : state.metaData()) {
if (previouslyWrittenIndices.contains(indexMetaData.getIndex()) && state.metaData().getIndices().get(indexMetaData.getIndex()).state().equals(IndexMetaData.State.CLOSE)) {
boolean isOrWasClosed = indexMetaData.state().equals(IndexMetaData.State.CLOSE);
// if the index is open we might still have to write the state if it just transitioned from closed to open
// so we have to check for that as well.
IndexMetaData previousMetaData = previousState.metaData().getIndices().get(indexMetaData.getIndex());
if (previousMetaData != null) {
isOrWasClosed = isOrWasClosed || previousMetaData.state().equals(IndexMetaData.State.CLOSE);
}
if (previouslyWrittenIndices.contains(indexMetaData.getIndex()) && isOrWasClosed) {
indices.add(indexMetaData.getIndex());
}
}

2
core/src/main/java/org/elasticsearch/gateway/LocalAllocateDangledIndices.java
View File

@ -128,7 +128,7 @@ public class LocalAllocateDangledIndices extends AbstractComponent {
if (currentState.metaData().hasIndex(indexMetaData.index())) {
continue;
}
if (currentState.metaData().aliases().containsKey(indexMetaData.index())) {
if (currentState.metaData().hasAlias(indexMetaData.index())) {
logger.warn("ignoring dangled index [{}] on node [{}] due to an existing alias with the same name",
indexMetaData.index(), request.fromNode);
continue;

9
core/src/main/java/org/elasticsearch/gateway/ReplicaShardAllocator.java
View File

@ -70,6 +70,10 @@ public abstract class ReplicaShardAllocator extends AbstractComponent {
if (shard.relocatingNodeId() != null) {
continue;
}
// if we are allocating a replica because of index creation, no need to go and find a copy, there isn't one...
if (shard.allocatedPostIndexCreate() == false) {
continue;
}
AsyncShardFetch.FetchResult<TransportNodesListShardStoreMetaData.NodeStoreFilesMetaData> shardStores = fetchData(shard, allocation);
if (shardStores.hasData() == false) {
@ -116,6 +120,11 @@ public abstract class ReplicaShardAllocator extends AbstractComponent {
continue;
}
// if we are allocating a replica because of index creation, no need to go and find a copy, there isn't one...
if (shard.allocatedPostIndexCreate() == false) {
continue;
}
// pre-check if it can be allocated to any node that currently exists, so we won't list the store for it for nothing
if (canBeAllocatedToAtLeastOneNode(shard, allocation) == false) {
logger.trace("{}: ignoring allocation, can't be allocated on any node", shard);

2
core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java
View File

@ -438,7 +438,7 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent<Indic
}
private boolean aliasesChanged(ClusterChangedEvent event) {
return !event.state().metaData().aliases().equals(event.previousState().metaData().aliases()) ||
return !event.state().metaData().equalsAliases(event.previousState().metaData()) ||
!event.state().routingTable().equals(event.previousState().routingTable());
}

4
core/src/main/java/org/elasticsearch/plugins/PluginsService.java
View File

@ -312,6 +312,10 @@ public class PluginsService extends AbstractComponent {
try (DirectoryStream<Path> stream = Files.newDirectoryStream(pluginsDirectory)) {
for (Path plugin : stream) {
try {
if (Files.isHidden(plugin)) {
logger.trace("--- skip hidden plugin file[{}]", plugin.toAbsolutePath());
continue;
}
logger.trace("--- adding plugin [{}]", plugin.toAbsolutePath());
PluginInfo info = PluginInfo.readFromProperties(plugin);
List<URL> urls = new ArrayList<>();

5
core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/ipv4/IPv4RangeBuilder.java
View File

@ -139,6 +139,10 @@ public class IPv4RangeBuilder extends AbstractRangeBuilder<IPv4RangeBuilder> {
int mask = (-1) << (32 - Integer.parseInt(parts[4]));
if (Integer.parseInt(parts[4]) == 0) {
mask = 0 << 32;
}
int from = addr & mask;
long longFrom = intIpToLongIp(from);
if (longFrom == 0) {
@ -147,6 +151,7 @@ public class IPv4RangeBuilder extends AbstractRangeBuilder<IPv4RangeBuilder> {
int to = from + (~mask);
long longTo = intIpToLongIp(to) + 1; // we have to +1 here as the range is non-inclusive on the "to" side
if (longTo == InternalIPv4Range.MAX_IP) {
longTo = -1;
}

21
core/src/test/java/org/elasticsearch/aliases/IndexAliasesTests.java
View File

@ -32,6 +32,7 @@ import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.metadata.AliasAction;
import org.elasticsearch.cluster.metadata.AliasMetaData;
import org.elasticsearch.cluster.metadata.AliasOrIndex;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.common.StopWatch;
import org.elasticsearch.common.settings.Settings;
@ -400,8 +401,8 @@ public class IndexAliasesTests extends ElasticsearchIntegrationTest {
assertThat(client().prepareCount("bars").setQuery(QueryBuilders.matchAllQuery()).get().getCount(), equalTo(1L));
}
@Test
public void testDeleteAliases() throws Exception {
logger.info("--> creating index [test1] and [test2]");
@ -421,17 +422,17 @@ public class IndexAliasesTests extends ElasticsearchIntegrationTest {
.addAlias("test2", "aliasToTests")
.addAlias("test2", "foos", termQuery("name", "foo"))
.addAlias("test2", "tests", termQuery("name", "test")));
String[] indices = {"test1", "test2"};
String[] indices = {"test1", "test2"};
String[] aliases = {"aliasToTest1", "foos", "bars", "tests", "aliasToTest2", "aliasToTests"};
admin().indices().prepareAliases().removeAlias(indices, aliases).get();
AliasesExistResponse response = admin().indices().prepareAliasesExist(aliases).get();
assertThat(response.exists(), equalTo(false));
}
@Test
public void testWaitForAliasCreationMultipleShards() throws Exception {
logger.info("--> creating index [test]");
@ -518,7 +519,7 @@ public class IndexAliasesTests extends ElasticsearchIntegrationTest {
assertThat(stopWatch.stop().lastTaskTime().millis(), lessThan(timeout.millis()));
logger.info("--> verify that filter was updated");
AliasMetaData aliasMetaData = internalCluster().clusterService().state().metaData().aliases().get("alias1").get("test");
AliasMetaData aliasMetaData = ((AliasOrIndex.Alias) internalCluster().clusterService().state().metaData().getAliasAndIndexLookup().get("alias1")).getFirstAliasMetaData();
assertThat(aliasMetaData.getFilter().toString(), equalTo("{\"term\":{\"name\":{\"value\":\"bar\",\"boost\":1.0}}}"));
logger.info("--> deleting alias1");
@ -526,9 +527,9 @@ public class IndexAliasesTests extends ElasticsearchIntegrationTest {
assertAcked((admin().indices().prepareAliases().removeAlias("test", "alias1").setTimeout(timeout)));
assertThat(stopWatch.stop().lastTaskTime().millis(), lessThan(timeout.millis()));
}
@Test(expected = AliasesNotFoundException.class)
public void testIndicesRemoveNonExistingAliasResponds404() throws Exception {
logger.info("--> creating index [test]");

6
core/src/test/java/org/elasticsearch/benchmark/aliases/AliasesBenchmark.java
View File

@ -26,6 +26,7 @@ import org.elasticsearch.cluster.metadata.AliasMetaData;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.indices.IndexAlreadyExistsException;
import org.elasticsearch.monitor.jvm.JvmStats;
import org.elasticsearch.node.Node;
import org.elasticsearch.node.NodeBuilder;
@ -39,7 +40,7 @@ public class AliasesBenchmark {
private final static String INDEX_NAME = "my-index";
public static void main(String[] args) throws IOException {
int NUM_ADDITIONAL_NODES = 0;
int NUM_ADDITIONAL_NODES = 1;
int BASE_ALIAS_COUNT = 100000;
int NUM_ADD_ALIAS_REQUEST = 1000;
@ -104,6 +105,7 @@ public class AliasesBenchmark {
if (i != numberOfAliases && i % 100 == 0) {
long avgTime = totalTime / 100;
System.out.println("Added [" + (i - numberOfAliases) + "] aliases. Avg create time: " + avgTime + " ms");
System.out.println("Heap used [" + JvmStats.jvmStats().getMem().getHeapUsed() + "]");
totalTime = 0;
}
@ -113,6 +115,8 @@ public class AliasesBenchmark {
.execute().actionGet();
totalTime += System.currentTimeMillis() - time;
}
System.gc();
System.out.println("Final heap used [" + JvmStats.jvmStats().getMem().getHeapUsed() + "]");
System.out.println("Number of aliases: " + countAliases(client));
client.close();

3
core/src/test/java/org/elasticsearch/cluster/ClusterStateDiffTests.java
View File

@ -51,6 +51,7 @@ import static org.elasticsearch.test.XContentTestUtils.convertToMap;
import static org.elasticsearch.test.XContentTestUtils.mapsEqualIgnoringArrayOrder;
import static org.elasticsearch.test.VersionUtils.randomVersion;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.is;
@ElasticsearchIntegrationTest.ClusterScope(scope = ElasticsearchIntegrationTest.Scope.SUITE, numDataNodes = 0, numClientNodes = 0)
@ -147,7 +148,7 @@ public class ClusterStateDiffTests extends ElasticsearchIntegrationTest {
assertThat(clusterStateFromDiffs.metaData().indices(), equalTo(clusterState.metaData().indices()));
assertThat(clusterStateFromDiffs.metaData().templates(), equalTo(clusterState.metaData().templates()));
assertThat(clusterStateFromDiffs.metaData().customs(), equalTo(clusterState.metaData().customs()));
assertThat(clusterStateFromDiffs.metaData().aliases(), equalTo(clusterState.metaData().aliases()));
assertThat(clusterStateFromDiffs.metaData().equalsAliases(clusterState.metaData()), is(true));
// JSON Serialization test - make sure that both states produce similar JSON
assertThat(mapsEqualIgnoringArrayOrder(convertToMap(clusterStateFromDiffs), convertToMap(clusterState)), equalTo(true));

3
core/src/test/java/org/elasticsearch/cluster/ack/AckTests.java
View File

@ -32,6 +32,7 @@ import org.elasticsearch.action.admin.indices.warmer.put.PutWarmerResponse;
import org.elasticsearch.client.Client;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.metadata.AliasMetaData;
import org.elasticsearch.cluster.metadata.AliasOrIndex;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.routing.ShardRouting;
import org.elasticsearch.cluster.routing.RoutingNode;
@ -310,7 +311,7 @@ public class AckTests extends ElasticsearchIntegrationTest {
assertAcked(client().admin().indices().prepareAliases().addAlias("test", "alias"));
for (Client client : clients()) {
AliasMetaData aliasMetaData = getLocalClusterState(client).metaData().aliases().get("alias").get("test");
AliasMetaData aliasMetaData = ((AliasOrIndex.Alias) getLocalClusterState(client).metaData().getAliasAndIndexLookup().get("alias")).getFirstAliasMetaData();
assertThat(aliasMetaData.alias(), equalTo("alias"));
}
}

16
core/src/test/java/org/elasticsearch/common/settings/loader/YamlSettingsLoaderTests.java
View File

@ -20,11 +20,11 @@
package org.elasticsearch.common.settings.loader;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.settings.SettingsException;
import org.elasticsearch.test.ElasticsearchTestCase;
import org.junit.Test;
import static org.elasticsearch.common.settings.Settings.settingsBuilder;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.equalTo;
/**
@ -49,4 +49,18 @@ public class YamlSettingsLoaderTests extends ElasticsearchTestCase {
assertThat(settings.getAsArray("test1.test3")[0], equalTo("test3-1"));
assertThat(settings.getAsArray("test1.test3")[1], equalTo("test3-2"));
}
@Test(expected = SettingsException.class)
public void testIndentation() {
settingsBuilder()
.loadFromClasspath("org/elasticsearch/common/settings/loader/indentation-settings.yml")
.build();
}
@Test(expected = SettingsException.class)
public void testIndentationWithExplicitDocumentStart() {
settingsBuilder()
.loadFromClasspath("org/elasticsearch/common/settings/loader/indentation-with-explicit-document-start-settings.yml")
.build();
}
}

10
core/src/test/java/org/elasticsearch/common/settings/loader/indentation-settings.yml Normal file
View File

@ -0,0 +1,10 @@
test1:
value1: value1
test2:
value2: value2
value3: 2
test3:
- test3-1
- test3-2
test4:
value4: value4

11
core/src/test/java/org/elasticsearch/common/settings/loader/indentation-with-explicit-document-start-settings.yml Normal file
View File

@ -0,0 +1,11 @@
test1:
value1: value1
test2:
value2: value2
value3: 2
test3:
- test3-1
- test3-2
---
test4:
value4: value4

52
core/src/test/java/org/elasticsearch/discovery/DiscoveryWithServiceDisruptionsTests.java
View File

@ -56,6 +56,7 @@ import org.elasticsearch.test.junit.annotations.TestLogging;
import org.elasticsearch.test.transport.MockTransportService;
import org.elasticsearch.transport.*;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
import java.io.IOException;
@ -65,6 +66,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope;
import static org.elasticsearch.test.ElasticsearchIntegrationTest.Scope;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
@ -237,7 +239,9 @@ public class DiscoveryWithServiceDisruptionsTests extends ElasticsearchIntegrati
}
/** Verify that nodes fault detection works after master (re) election */
/**
* Verify that nodes fault detection works after master (re) election
*/
@Test
public void testNodesFDAfterMasterReelection() throws Exception {
startCluster(4);
@ -414,7 +418,7 @@ public class DiscoveryWithServiceDisruptionsTests extends ElasticsearchIntegrati
/**
* Test that we do not loose document whose indexing request was successful, under a randomly selected disruption scheme
* We also collect & report the type of indexing failures that occur.
*
* <p/>
* This test is a superset of tests run in the Jepsen test suite, with the exception of versioned updates
*/
@Test
@ -948,6 +952,50 @@ public class DiscoveryWithServiceDisruptionsTests extends ElasticsearchIntegrati
ensureStableCluster(3);
}
@Test
public void testIndexImportedFromDataOnlyNodesIfMasterLostDataFolder() throws Exception {
// test for https://github.com/elastic/elasticsearch/issues/8823
configureCluster(2, 1);
String masterNode = internalCluster().startMasterOnlyNode(Settings.EMPTY);
internalCluster().startDataOnlyNode(Settings.EMPTY);
ensureStableCluster(2);
assertAcked(prepareCreate("index").setSettings(Settings.builder().put("index.number_of_replicas", 0)));
index("index", "doc", "1", jsonBuilder().startObject().field("text", "some text").endObject());
ensureGreen();
internalCluster().restartNode(masterNode, new InternalTestCluster.RestartCallback() {
public boolean clearData(String nodeName) {
return true;
}
});
ensureGreen("index");
assertTrue(client().prepareGet("index", "doc", "1").get().isExists());
}
// tests if indices are really deleted even if a master transition inbetween
@Ignore("https://github.com/elastic/elasticsearch/issues/11665")
@Test
public void testIndicesDeleted() throws Exception {
configureCluster(3, 2);
Future<List<String>> masterNodes= internalCluster().startMasterOnlyNodesAsync(2);
Future<String> dataNode = internalCluster().startDataOnlyNodeAsync();
dataNode.get();
masterNodes.get();
ensureStableCluster(3);
assertAcked(prepareCreate("test"));
ensureYellow();
String masterNode1 = internalCluster().getMasterName();
NetworkPartition networkPartition = new NetworkUnresponsivePartition(masterNode1, dataNode.get(), getRandom());
internalCluster().setDisruptionScheme(networkPartition);
networkPartition.startDisrupting();
internalCluster().client(masterNode1).admin().indices().prepareDelete("test").setTimeout("1s").get();
internalCluster().restartNode(masterNode1, InternalTestCluster.EMPTY_CALLBACK);
ensureYellow();
assertFalse(client().admin().indices().prepareExists("test").get().isExists());
}
protected NetworkPartition addRandomPartition() {
NetworkPartition partition;

4
core/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java
View File

@ -174,9 +174,9 @@ public class GatewayMetaStateTests extends ElasticsearchAllocationTestCase {
if (stateInMemory) {
inMemoryMetaData = event.previousState().metaData();
ImmutableSet.Builder<String> relevantIndices = ImmutableSet.builder();
oldIndicesList = relevantIndices.addAll(GatewayMetaState.getRelevantIndices(event.previousState(), oldIndicesList)).build();
oldIndicesList = relevantIndices.addAll(GatewayMetaState.getRelevantIndices(event.previousState(), event.previousState(), oldIndicesList)).build();
}
Set<String> newIndicesList = GatewayMetaState.getRelevantIndices(event.state(), oldIndicesList);
Set<String> newIndicesList = GatewayMetaState.getRelevantIndices(event.state(),event.previousState(), oldIndicesList);
// third, get the actual write info
Iterator<GatewayMetaState.IndexMetaWriteInfo> indices = GatewayMetaState.resolveStatesToBeWritten(oldIndicesList, newIndicesList, inMemoryMetaData, event.state().metaData()).iterator();

301
core/src/test/java/org/elasticsearch/gateway/MetaDataWriteDataNodesTests.java
View File

@ -21,6 +21,7 @@ package org.elasticsearch.gateway;
import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
import com.google.common.base.Predicate;
import org.apache.lucene.util.LuceneTestCase;
import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
import org.elasticsearch.cluster.metadata.IndexMetaData;
@ -33,227 +34,81 @@ import org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope;
import org.elasticsearch.test.InternalTestCluster;
import org.junit.Test;
import java.io.IOException;
import java.util.LinkedHashMap;
import java.util.concurrent.Future;
import static org.elasticsearch.client.Requests.clusterHealthRequest;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.elasticsearch.test.ElasticsearchIntegrationTest.Scope;
import static org.elasticsearch.test.InternalTestCluster.RestartCallback;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
import static org.hamcrest.Matchers.equalTo;
/**
*
*/
@LuceneTestCase.Slow
@ClusterScope(scope = Scope.TEST, numDataNodes = 0)
public class MetaDataWriteDataNodesTests extends ElasticsearchIntegrationTest {
@Test
public void testMetaWrittenAlsoOnDataNode() throws Exception {
// this test checks that index state is written on data only nodes
String masterNodeName = startMasterNode();
String redNode = startDataNode("red");
assertAcked(prepareCreate("test").setSettings(Settings.builder().put("index.number_of_replicas", 0)));
// this test checks that index state is written on data only nodes if they have a shard allocated
String masterNode = internalCluster().startMasterOnlyNode(Settings.EMPTY);
String dataNode = internalCluster().startDataOnlyNode(Settings.EMPTY);
assertAcked(prepareCreate("test").setSettings("index.number_of_replicas", 0));
index("test", "doc", "1", jsonBuilder().startObject().field("text", "some text").endObject());
ensureGreen("test");
assertIndexInMetaState(redNode, "test");
assertIndexInMetaState(masterNodeName, "test");
//stop master node and start again with an empty data folder
((InternalTestCluster) cluster()).stopCurrentMasterNode();
String newMasterNode = startMasterNode();
ensureGreen("test");
// check for meta data
assertIndexInMetaState(redNode, "test");
assertIndexInMetaState(newMasterNode, "test");
// check if index and doc is still there
ensureGreen("test");
assertTrue(client().prepareGet("test", "doc", "1").get().isExists());
}
@Test
public void testMetaWrittenOnlyForIndicesOnNodesThatHaveAShard() throws Exception {
// this test checks that the index state is only written to a data only node if they have a shard of that index allocated on the node
String masterNode = startMasterNode();
String blueNode = startDataNode("blue");
String redNode = startDataNode("red");
assertAcked(prepareCreate("blue_index").setSettings(Settings.builder().put("index.number_of_replicas", 0).put(FilterAllocationDecider.INDEX_ROUTING_INCLUDE_GROUP + "color", "blue")));
index("blue_index", "doc", "1", jsonBuilder().startObject().field("text", "some text").endObject());
assertAcked(prepareCreate("red_index").setSettings(Settings.builder().put("index.number_of_replicas", 0).put(FilterAllocationDecider.INDEX_ROUTING_INCLUDE_GROUP + "color", "red")));
index("red_index", "doc", "1", jsonBuilder().startObject().field("text", "some text").endObject());
ensureGreen();
assertIndexNotInMetaState(blueNode, "red_index");
assertIndexNotInMetaState(redNode, "blue_index");
assertIndexInMetaState(blueNode, "blue_index");
assertIndexInMetaState(redNode, "red_index");
assertIndexInMetaState(masterNode, "red_index");
assertIndexInMetaState(masterNode, "blue_index");
// not the index state for blue_index should only be written on blue_node and the for red_index only on red_node
// we restart red node and master but with empty data folders
stopNode(redNode);
((InternalTestCluster) cluster()).stopCurrentMasterNode();
masterNode = startMasterNode();
redNode = startDataNode("red");
ensureGreen();
assertIndexNotInMetaState(blueNode, "red_index");
assertIndexInMetaState(blueNode, "blue_index");
assertIndexNotInMetaState(redNode, "red_index");
assertIndexNotInMetaState(redNode, "blue_index");
assertIndexNotInMetaState(masterNode, "red_index");
assertIndexInMetaState(masterNode, "blue_index");
// check that blue index is still there
assertFalse(client().admin().indices().prepareExists("red_index").get().isExists());
assertTrue(client().prepareGet("blue_index", "doc", "1").get().isExists());
// red index should be gone
// if the blue node had stored the index state then cluster health would be red and red_index would exist
assertFalse(client().admin().indices().prepareExists("red_index").get().isExists());
assertIndexInMetaState(dataNode, "test");
assertIndexInMetaState(masterNode, "test");
}
@Test
public void testMetaIsRemovedIfAllShardsFromIndexRemoved() throws Exception {
// this test checks that the index state is removed from a data only node once all shards have been allocated away from it
String masterNode = startMasterNode();
String blueNode = startDataNode("blue");
String redNode = startDataNode("red");
// create blue_index on blue_node and same for red
client().admin().cluster().health(clusterHealthRequest().waitForYellowStatus().waitForNodes("3")).get();
assertAcked(prepareCreate("blue_index").setSettings(Settings.builder().put("index.number_of_replicas", 0).put(FilterAllocationDecider.INDEX_ROUTING_INCLUDE_GROUP + "color", "blue")));
index("blue_index", "doc", "1", jsonBuilder().startObject().field("text", "some text").endObject());
assertAcked(prepareCreate("red_index").setSettings(Settings.builder().put("index.number_of_replicas", 0).put(FilterAllocationDecider.INDEX_ROUTING_INCLUDE_GROUP + "color", "red")));
index("red_index", "doc", "1", jsonBuilder().startObject().field("text", "some text").endObject());
String masterNode = internalCluster().startMasterOnlyNode(Settings.EMPTY);
Future<String> nodeName1 = internalCluster().startDataOnlyNodeAsync();
Future<String> nodeName2 = internalCluster().startDataOnlyNodeAsync();
String node1 = nodeName1.get();
String node2 = nodeName2.get();
String index = "index";
assertAcked(prepareCreate(index).setSettings(Settings.builder().put("index.number_of_replicas", 0).put(FilterAllocationDecider.INDEX_ROUTING_INCLUDE_GROUP + "_name", node1)));
index(index, "doc", "1", jsonBuilder().startObject().field("text", "some text").endObject());
ensureGreen();
assertIndexNotInMetaState(redNode, "blue_index");
assertIndexNotInMetaState(blueNode, "red_index");
assertIndexInMetaState(redNode, "red_index");
assertIndexInMetaState(blueNode, "blue_index");
assertIndexInMetaState(masterNode, "red_index");
assertIndexInMetaState(masterNode, "blue_index");
assertIndexInMetaState(node1, index);
assertIndexNotInMetaState(node2, index);
assertIndexInMetaState(masterNode, index);
// now relocate blue_index to red_node and red_index to blue_node
logger.debug("relocating indices...");
client().admin().indices().prepareUpdateSettings("blue_index").setSettings(Settings.builder().put(FilterAllocationDecider.INDEX_ROUTING_INCLUDE_GROUP + "color", "red")).get();
client().admin().indices().prepareUpdateSettings("red_index").setSettings(Settings.builder().put(FilterAllocationDecider.INDEX_ROUTING_INCLUDE_GROUP + "color", "blue")).get();
logger.debug("relocating index...");
client().admin().indices().prepareUpdateSettings(index).setSettings(Settings.builder().put(FilterAllocationDecider.INDEX_ROUTING_INCLUDE_GROUP + "_name", node2)).get();
client().admin().cluster().prepareHealth().setWaitForRelocatingShards(0).get();
ensureGreen();
assertIndexNotInMetaState(redNode, "red_index");
assertIndexNotInMetaState(blueNode, "blue_index");
assertIndexInMetaState(redNode, "blue_index");
assertIndexInMetaState(blueNode, "red_index");
assertIndexInMetaState(masterNode, "red_index");
assertIndexInMetaState(masterNode, "blue_index");
//at this point the blue_index is on red node and the red_index on blue node
// now, when we start red and master node again but without data folder, the red index should be gone but the blue index should initialize fine
stopNode(redNode);
((InternalTestCluster) cluster()).stopCurrentMasterNode();
masterNode = startMasterNode();
redNode = startDataNode("red");
ensureGreen();
assertIndexNotInMetaState(redNode, "blue_index");
assertIndexNotInMetaState(blueNode, "blue_index");
assertIndexNotInMetaState(redNode, "red_index");
assertIndexInMetaState(blueNode, "red_index");
assertIndexInMetaState(masterNode, "red_index");
assertIndexNotInMetaState(masterNode, "blue_index");
assertTrue(client().prepareGet("red_index", "doc", "1").get().isExists());
// if the red_node had stored the index state then cluster health would be red and blue_index would exist
assertFalse(client().admin().indices().prepareExists("blue_index").get().isExists());
}
@Test
public void testMetaWrittenWhenIndexIsClosed() throws Exception {
String masterNode = startMasterNode();
String redNodeDataPath = createTempDir().toString();
String redNode = startDataNode("red", redNodeDataPath);
String blueNode = startDataNode("blue");
// create red_index on red_node and same for red
client().admin().cluster().health(clusterHealthRequest().waitForYellowStatus().waitForNodes("3")).get();
assertAcked(prepareCreate("red_index").setSettings(Settings.builder().put("index.number_of_replicas", 0).put(FilterAllocationDecider.INDEX_ROUTING_INCLUDE_GROUP + "color", "red")));
index("red_index", "doc", "1", jsonBuilder().startObject().field("text", "some text").endObject());
ensureGreen();
assertIndexNotInMetaState(blueNode, "red_index");
assertIndexInMetaState(redNode, "red_index");
assertIndexInMetaState(masterNode, "red_index");
client().admin().indices().prepareClose("red_index").get();
// close the index
ClusterStateResponse clusterStateResponse = client().admin().cluster().prepareState().get();
assertThat(clusterStateResponse.getState().getMetaData().index("red_index").getState().name(), equalTo(IndexMetaData.State.CLOSE.name()));
// restart master with empty data folder and maybe red node
boolean restartRedNode = randomBoolean();
//at this point the red_index on red node
if (restartRedNode) {
stopNode(redNode);
}
((InternalTestCluster) cluster()).stopCurrentMasterNode();
masterNode = startMasterNode();
if (restartRedNode) {
redNode = startDataNode("red", redNodeDataPath);
}
ensureGreen("red_index");
assertIndexNotInMetaState(blueNode, "red_index");
assertIndexInMetaState(redNode, "red_index");
assertIndexInMetaState(masterNode, "red_index");
clusterStateResponse = client().admin().cluster().prepareState().get();
assertThat(clusterStateResponse.getState().getMetaData().index("red_index").getState().name(), equalTo(IndexMetaData.State.CLOSE.name()));
// open the index again
client().admin().indices().prepareOpen("red_index").get();
clusterStateResponse = client().admin().cluster().prepareState().get();
assertThat(clusterStateResponse.getState().getMetaData().index("red_index").getState().name(), equalTo(IndexMetaData.State.OPEN.name()));
// restart again
ensureGreen();
if (restartRedNode) {
stopNode(redNode);
}
((InternalTestCluster) cluster()).stopCurrentMasterNode();
masterNode = startMasterNode();
if (restartRedNode) {
redNode = startDataNode("red", redNodeDataPath);
}
ensureGreen("red_index");
assertIndexNotInMetaState(blueNode, "red_index");
assertIndexInMetaState(redNode, "red_index");
assertIndexInMetaState(masterNode, "red_index");
clusterStateResponse = client().admin().cluster().prepareState().get();
assertThat(clusterStateResponse.getState().getMetaData().index("red_index").getState().name(), equalTo(IndexMetaData.State.OPEN.name()));
assertTrue(client().prepareGet("red_index", "doc", "1").get().isExists());
assertIndexNotInMetaState(node1, index);
assertIndexInMetaState(node2, index);
assertIndexInMetaState(masterNode, index);
}
@Test
public void testMetaWrittenWhenIndexIsClosedAndMetaUpdated() throws Exception {
String masterNode = startMasterNode();
String redNodeDataPath = createTempDir().toString();
String redNode = startDataNode("red", redNodeDataPath);
// create red_index on red_node and same for red
client().admin().cluster().health(clusterHealthRequest().waitForYellowStatus().waitForNodes("2")).get();
assertAcked(prepareCreate("red_index").setSettings(Settings.builder().put("index.number_of_replicas", 0).put(FilterAllocationDecider.INDEX_ROUTING_INCLUDE_GROUP + "color", "red")));
index("red_index", "doc", "1", jsonBuilder().startObject().field("text", "some text").endObject());
String masterNode = internalCluster().startMasterOnlyNode(Settings.EMPTY);
final String dataNode = internalCluster().startDataOnlyNode(Settings.EMPTY);
logger.info("--> wait for green red_index");
final String index = "index";
assertAcked(prepareCreate(index).setSettings(Settings.builder().put("index.number_of_replicas", 0)));
logger.info("--> wait for green index");
ensureGreen();
logger.info("--> wait for meta state written for red_index");
assertIndexInMetaState(redNode, "red_index");
assertIndexInMetaState(masterNode, "red_index");
logger.info("--> wait for meta state written for index");
assertIndexInMetaState(dataNode, index);
assertIndexInMetaState(masterNode, index);
logger.info("--> close red_index");
client().admin().indices().prepareClose("red_index").get();
logger.info("--> close index");
client().admin().indices().prepareClose(index).get();
// close the index
ClusterStateResponse clusterStateResponse = client().admin().cluster().prepareState().get();
assertThat(clusterStateResponse.getState().getMetaData().index("red_index").getState().name(), equalTo(IndexMetaData.State.CLOSE.name()));
assertThat(clusterStateResponse.getState().getMetaData().index(index).getState().name(), equalTo(IndexMetaData.State.CLOSE.name()));
logger.info("--> restart red node");
stopNode(redNode);
redNode = startDataNode("red", redNodeDataPath);
client().admin().indices().preparePutMapping("red_index").setType("doc").setSource(jsonBuilder().startObject()
// update the mapping. this should cause the new meta data to be written although index is closed
client().admin().indices().preparePutMapping(index).setType("doc").setSource(jsonBuilder().startObject()
.startObject("properties")
.startObject("integer_field")
.field("type", "integer")
@ -261,45 +116,43 @@ public class MetaDataWriteDataNodesTests extends ElasticsearchIntegrationTest {
.endObject()
.endObject()).get();
GetMappingsResponse getMappingsResponse = client().admin().indices().prepareGetMappings("red_index").addTypes("doc").get();
assertNotNull(((LinkedHashMap) (getMappingsResponse.getMappings().get("red_index").get("doc").getSourceAsMap().get("properties"))).get("integer_field"));
// restart master with empty data folder and maybe red node
((InternalTestCluster) cluster()).stopCurrentMasterNode();
masterNode = startMasterNode();
GetMappingsResponse getMappingsResponse = client().admin().indices().prepareGetMappings(index).addTypes("doc").get();
assertNotNull(((LinkedHashMap) (getMappingsResponse.getMappings().get(index).get("doc").getSourceAsMap().get("properties"))).get("integer_field"));
ensureGreen("red_index");
assertIndexInMetaState(redNode, "red_index");
assertIndexInMetaState(masterNode, "red_index");
clusterStateResponse = client().admin().cluster().prepareState().get();
assertThat(clusterStateResponse.getState().getMetaData().index("red_index").getState().name(), equalTo(IndexMetaData.State.CLOSE.name()));
getMappingsResponse = client().admin().indices().prepareGetMappings("red_index").addTypes("doc").get();
assertNotNull(((LinkedHashMap) (getMappingsResponse.getMappings().get("red_index").get("doc").getSourceAsMap().get("properties"))).get("integer_field"));
// make sure it was also written on red node although index is closed
ImmutableOpenMap<String, IndexMetaData> indicesMetaData = getIndicesMetaDataOnNode(dataNode);
assertNotNull(((LinkedHashMap) (indicesMetaData.get(index).getMappings().get("doc").getSourceAsMap().get("properties"))).get("integer_field"));
assertThat(indicesMetaData.get(index).state(), equalTo(IndexMetaData.State.CLOSE));
}
/* Try the same and see if this also works if node was just restarted.
* Each node holds an array of indices it knows of and checks if it should
* write new meta data by looking up in this array. We need it because if an
* index is closed it will not appear in the shard routing and we therefore
* need to keep track of what we wrote before. However, when the node is
* restarted this array is empty and we have to fill it before we decide
* what we write. This is why we explicitly test for it.
*/
internalCluster().restartNode(dataNode, new RestartCallback());
client().admin().indices().preparePutMapping(index).setType("doc").setSource(jsonBuilder().startObject()
.startObject("properties")
.startObject("float_field")
.field("type", "float")
.endObject()
.endObject()
.endObject()).get();
private String startDataNode(String color) {
return startDataNode(color, createTempDir().toString());
}
getMappingsResponse = client().admin().indices().prepareGetMappings(index).addTypes("doc").get();
assertNotNull(((LinkedHashMap) (getMappingsResponse.getMappings().get(index).get("doc").getSourceAsMap().get("properties"))).get("float_field"));
private String startDataNode(String color, String newDataPath) {
Settings.Builder settingsBuilder = Settings.builder()
.put("node.data", true)
.put("node.master", false)
.put("node.color", color)
.put("path.data", newDataPath);
return internalCluster().startNode(settingsBuilder.build());
}
// make sure it was also written on red node although index is closed
indicesMetaData = getIndicesMetaDataOnNode(dataNode);
assertNotNull(((LinkedHashMap) (indicesMetaData.get(index).getMappings().get("doc").getSourceAsMap().get("properties"))).get("float_field"));
assertThat(indicesMetaData.get(index).state(), equalTo(IndexMetaData.State.CLOSE));
private String startMasterNode() {
Settings.Builder settingsBuilder = Settings.builder()
.put("node.data", false)
.put("node.master", true)
.put("path.data", createTempDir().toString());
return internalCluster().startNode(settingsBuilder.build());
}
private void stopNode(String name) throws IOException {
internalCluster().stopRandomNode(InternalTestCluster.nameFilter(name));
// finally check that meta data is also written of index opened again
assertAcked(client().admin().indices().prepareOpen(index).get());
indicesMetaData = getIndicesMetaDataOnNode(dataNode);
assertThat(indicesMetaData.get(index).state(), equalTo(IndexMetaData.State.OPEN));
}
protected void assertIndexNotInMetaState(String nodeName, String indexName) throws Exception {
@ -335,14 +188,18 @@ public class MetaDataWriteDataNodesTests extends ElasticsearchIntegrationTest {
}
private boolean metaStateExists(String nodeName, String indexName) throws Exception {
GatewayMetaState nodeMetaState = ((InternalTestCluster) cluster()).getInstance(GatewayMetaState.class, nodeName);
MetaData nodeMetaData = null;
nodeMetaData = nodeMetaState.loadMetaState();
ImmutableOpenMap<String, IndexMetaData> indices = nodeMetaData.getIndices();
ImmutableOpenMap<String, IndexMetaData> indices = getIndicesMetaDataOnNode(nodeName);
boolean inMetaSate = false;
for (ObjectObjectCursor<String, IndexMetaData> index : indices) {
inMetaSate = inMetaSate || index.key.equals(indexName);
}
return inMetaSate;
}
private ImmutableOpenMap<String, IndexMetaData> getIndicesMetaDataOnNode(String nodeName) throws Exception {
GatewayMetaState nodeMetaState = ((InternalTestCluster) cluster()).getInstance(GatewayMetaState.class, nodeName);
MetaData nodeMetaData = null;
nodeMetaData = nodeMetaState.loadMetaState();
return nodeMetaData.getIndices();
}
}

40
core/src/test/java/org/elasticsearch/gateway/ReplicaShardAllocatorTests.java
View File

@ -19,6 +19,7 @@
package org.elasticsearch.gateway;
import com.carrotsearch.randomizedtesting.generators.RandomPicks;
import org.elasticsearch.Version;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.metadata.IndexMetaData;
@ -42,8 +43,10 @@ import org.junit.Before;
import org.junit.Test;
import java.util.Collections;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.atomic.AtomicBoolean;
import static org.hamcrest.Matchers.equalTo;
@ -75,6 +78,33 @@ public class ReplicaShardAllocatorTests extends ElasticsearchAllocationTestCase
assertThat(allocation.routingNodes().unassigned().ignored().get(0).shardId(), equalTo(shardId));
}
/**
* Verifies that on index creation, we don't go and fetch data, but keep the replica shard unassigned to let
* the shard allocator to allocate it. There isn't a copy around to find anyhow.
*/
@Test
public void testNoAsyncFetchOnIndexCreation() {
RoutingAllocation allocation = onePrimaryOnNode1And1Replica(yesAllocationDeciders(), Settings.EMPTY, UnassignedInfo.Reason.INDEX_CREATED);
testAllocator.clean();
testAllocator.allocateUnassigned(allocation);
assertThat(testAllocator.getFetchDataCalledAndClean(), equalTo(false));
assertThat(allocation.routingNodes().shardsWithState(ShardRoutingState.UNASSIGNED).size(), equalTo(1));
assertThat(allocation.routingNodes().shardsWithState(ShardRoutingState.UNASSIGNED).get(0).shardId(), equalTo(shardId));
}
/**
* Verifies that for anything but index creation, fetch data ends up being called, since we need to go and try
* and find a better copy for the shard.
*/
@Test
public void testAsyncFetchOnAnythingButIndexCreation() {
UnassignedInfo.Reason reason = RandomPicks.randomFrom(getRandom(), EnumSet.complementOf(EnumSet.of(UnassignedInfo.Reason.INDEX_CREATED)));
RoutingAllocation allocation = onePrimaryOnNode1And1Replica(yesAllocationDeciders(), Settings.EMPTY, reason);
testAllocator.clean();
testAllocator.allocateUnassigned(allocation);
assertThat("failed with reason " + reason, testAllocator.getFetchDataCalledAndClean(), equalTo(true));
}
/**
* Verifies that when there is a full match (syncId and files) we allocate it to matching node.
*/
@ -253,7 +283,7 @@ public class ReplicaShardAllocatorTests extends ElasticsearchAllocationTestCase
}
private RoutingAllocation onePrimaryOnNode1And1Replica(AllocationDeciders deciders) {
return onePrimaryOnNode1And1Replica(deciders, Settings.EMPTY, UnassignedInfo.Reason.INDEX_CREATED);
return onePrimaryOnNode1And1Replica(deciders, Settings.EMPTY, UnassignedInfo.Reason.CLUSTER_RECOVERED);
}
private RoutingAllocation onePrimaryOnNode1And1Replica(AllocationDeciders deciders, Settings settings, UnassignedInfo.Reason reason) {
@ -283,7 +313,7 @@ public class ReplicaShardAllocatorTests extends ElasticsearchAllocationTestCase
.add(IndexRoutingTable.builder(shardId.getIndex())
.addIndexShard(new IndexShardRoutingTable.Builder(shardId)
.addShard(TestShardRouting.newShardRouting(shardId.getIndex(), shardId.getId(), node1.id(), true, ShardRoutingState.STARTED, 10))
.addShard(TestShardRouting.newShardRouting(shardId.getIndex(), shardId.getId(), node2.id(), false, ShardRoutingState.INITIALIZING, 10))
.addShard(TestShardRouting.newShardRouting(shardId.getIndex(), shardId.getId(), node2.id(), null, null, false, ShardRoutingState.INITIALIZING, 10, new UnassignedInfo(UnassignedInfo.Reason.CLUSTER_RECOVERED, null)))
.build())
)
.build();
@ -297,6 +327,7 @@ public class ReplicaShardAllocatorTests extends ElasticsearchAllocationTestCase
class TestAllocator extends ReplicaShardAllocator {
private Map<DiscoveryNode, TransportNodesListShardStoreMetaData.StoreFilesMetaData> data = null;
private AtomicBoolean fetchDataCalled = new AtomicBoolean(false);
public TestAllocator() {
super(Settings.EMPTY);
@ -310,6 +341,10 @@ public class ReplicaShardAllocatorTests extends ElasticsearchAllocationTestCase
data = new HashMap<>();
}
public boolean getFetchDataCalledAndClean() {
return fetchDataCalled.getAndSet(false);
}
public TestAllocator addData(DiscoveryNode node, boolean allocated, String syncId, StoreFileMetaData... files) {
if (data == null) {
data = new HashMap<>();
@ -328,6 +363,7 @@ public class ReplicaShardAllocatorTests extends ElasticsearchAllocationTestCase
@Override
protected AsyncShardFetch.FetchResult<TransportNodesListShardStoreMetaData.NodeStoreFilesMetaData> fetchData(ShardRouting shard, RoutingAllocation allocation) {
fetchDataCalled.set(true);
Map<DiscoveryNode, TransportNodesListShardStoreMetaData.NodeStoreFilesMetaData> tData = null;
if (data != null) {
tData = new HashMap<>();

74
core/src/test/java/org/elasticsearch/search/aggregations/bucket/IPv4RangeTests.java
View File

@ -83,6 +83,33 @@ public class IPv4RangeTests extends ElasticsearchIntegrationTest {
}
indexRandom(true, builders.toArray(new IndexRequestBuilder[builders.size()]));
}
{
assertAcked(prepareCreate("range_idx")
.addMapping("type", "ip", "type=ip", "ips", "type=ip"));
IndexRequestBuilder[] builders = new IndexRequestBuilder[4];
builders[0] = client().prepareIndex("range_idx", "type").setSource(jsonBuilder()
.startObject()
.field("ip", "0.0.0.0")
.endObject());
builders[1] = client().prepareIndex("range_idx", "type").setSource(jsonBuilder()
.startObject()
.field("ip", "0.0.0.255")
.endObject());
builders[2] = client().prepareIndex("range_idx", "type").setSource(jsonBuilder()
.startObject()
.field("ip", "255.255.255.0")
.endObject());
builders[3] = client().prepareIndex("range_idx", "type").setSource(jsonBuilder()
.startObject()
.field("ip", "255.255.255.255")
.endObject());
indexRandom(true, builders);
}
ensureSearchable();
}
@ -869,4 +896,51 @@ public class IPv4RangeTests extends ElasticsearchIntegrationTest {
assertThat(buckets.get(0).getToAsString(), equalTo("10.0.0.10"));
assertThat(buckets.get(0).getDocCount(), equalTo(0l));
}
@Test
public void mask0() {
SearchResponse response = client().prepareSearch("idx")
.addAggregation(ipRange("range")
.field("ip")
.addMaskRange("0.0.0.0/0"))
.execute().actionGet();
assertSearchResponse(response);
Range range = response.getAggregations().get("range");
assertThat(range, notNullValue());
assertThat(range.getName(), equalTo("range"));
List<? extends Bucket> buckets = range.getBuckets();
assertThat(range.getBuckets().size(), equalTo(1));
Range.Bucket bucket = buckets.get(0);
assertThat((String) bucket.getKey(), equalTo("0.0.0.0/0"));
assertThat(bucket.getFromAsString(), nullValue());
assertThat(bucket.getToAsString(), nullValue());
assertThat(((Number) bucket.getTo()).doubleValue(), equalTo(Double.POSITIVE_INFINITY));
assertEquals(255l, bucket.getDocCount());
}
@Test
public void mask0SpecialIps() {
SearchResponse response = client().prepareSearch("range_idx")
.addAggregation(ipRange("range")
.field("ip")
.addMaskRange("0.0.0.0/0"))
.execute().actionGet();
assertSearchResponse(response);
Range range = response.getAggregations().get("range");
assertThat(range, notNullValue());
assertThat(range.getName(), equalTo("range"));
List<? extends Bucket> buckets = range.getBuckets();
assertThat(range.getBuckets().size(), equalTo(1));
Range.Bucket bucket = buckets.get(0);
assertEquals(4l, bucket.getDocCount());
}
}

62
core/src/test/java/org/elasticsearch/test/InternalTestCluster.java
View File

@ -1284,6 +1284,18 @@ public final class InternalTestCluster extends TestCluster {
}
}
/**
* Restarts a node and calls the callback during restart.
*/
public void restartNode(String nodeName, RestartCallback callback) throws Exception {
ensureOpen();
NodeAndClient nodeAndClient = nodes.get(nodeName);
if (nodeAndClient != null) {
logger.info("Restarting node [{}] ", nodeAndClient.name);
nodeAndClient.restart(callback);
}
}
private void restartAllNodes(boolean rollingRestart, RestartCallback callback) throws Exception {
ensureOpen();
List<NodeAndClient> toRemove = new ArrayList<>();
@ -1341,7 +1353,7 @@ public final class InternalTestCluster extends TestCluster {
}
private static final RestartCallback EMPTY_CALLBACK = new RestartCallback() {
public static final RestartCallback EMPTY_CALLBACK = new RestartCallback() {
@Override
public Settings onNodeStopped(String node) {
return null;
@ -1466,6 +1478,52 @@ public final class InternalTestCluster extends TestCluster {
return buildNode.name;
}
public synchronized ListenableFuture<List<String>> startMasterOnlyNodesAsync(int numNodes) {
return startMasterOnlyNodesAsync(numNodes, Settings.EMPTY);
}
public synchronized ListenableFuture<List<String>> startMasterOnlyNodesAsync(int numNodes, Settings settings) {
Settings settings1 = Settings.builder().put(settings).put("node.master", true).put("node.data", false).build();
return startNodesAsync(numNodes, settings1, Version.CURRENT);
}
public synchronized ListenableFuture<List<String>> startDataOnlyNodesAsync(int numNodes) {
return startDataOnlyNodesAsync(numNodes, Settings.EMPTY);
}
public synchronized ListenableFuture<List<String>> startDataOnlyNodesAsync(int numNodes, Settings settings) {
Settings settings1 = Settings.builder().put(settings).put("node.master", false).put("node.data", true).build();
return startNodesAsync(numNodes, settings1, Version.CURRENT);
}
public synchronized ListenableFuture<String> startMasterOnlyNodeAsync() {
return startMasterOnlyNodeAsync(Settings.EMPTY);
}
public synchronized ListenableFuture<String> startMasterOnlyNodeAsync(Settings settings) {
Settings settings1 = Settings.builder().put(settings).put("node.master", true).put("node.data", false).build();
return startNodeAsync(settings1, Version.CURRENT);
}
public synchronized String startMasterOnlyNode(Settings settings) {
Settings settings1 = Settings.builder().put(settings).put("node.master", true).put("node.data", false).build();
return startNode(settings1, Version.CURRENT);
}
public synchronized ListenableFuture<String> startDataOnlyNodeAsync() {
return startDataOnlyNodeAsync(Settings.EMPTY);
}
public synchronized ListenableFuture<String> startDataOnlyNodeAsync(Settings settings) {
Settings settings1 = Settings.builder().put(settings).put("node.master", false).put("node.data", true).build();
return startNodeAsync(settings1, Version.CURRENT);
}
public synchronized String startDataOnlyNode(Settings settings) {
Settings settings1 = Settings.builder().put(settings).put("node.master", false).put("node.data", true).build();
return startNode(settings1, Version.CURRENT);
}
/**
* Starts a node in an async manner with the given settings and returns future with its name.
*/
@ -1724,7 +1782,7 @@ public final class InternalTestCluster extends TestCluster {
* and / or {@link #fullRestart(InternalTestCluster.RestartCallback)} to execute actions at certain
* stages of the restart.
*/
public static abstract class RestartCallback {
public static class RestartCallback {
/**
* Executed once the give node name has been stopped.