Improved how aliases are handled in the cluster state.

The following changes improved alias creation:
* Moved away from ImmutableMap to JCF's UnmodifiableMap. The ImmutableMap always made a copy, whereas the UnmodifiableMap just wraps the target map.
* Reducing the number of maps being created during the creation of MetaData and IndexMetadata.
* Changed IndexAliasesService's aliases from a copy on write ImmutableMap to ConcurrentMap.

Closes #3410
This commit is contained in:
Martijn van Groningen 2013-07-30 17:06:55 +02:00
parent 33d8571d1e
commit a235a55943
11 changed files with 312 additions and 158 deletions

View File

@ -143,6 +143,10 @@ public class IndicesAliasesRequest extends MasterNodeOperationRequest<IndicesAli
return this.aliasActions; return this.aliasActions;
} }
public List<AliasAction> getAliasActions() {
return aliasActions();
}
/** /**
* Timeout to wait till the put mapping gets acknowledged of all current cluster nodes. Defaults to * Timeout to wait till the put mapping gets acknowledged of all current cluster nodes. Defaults to
* <tt>10s</tt>. * <tt>10s</tt>.

View File

@ -19,7 +19,6 @@
package org.elasticsearch.action.admin.indices.exists.types; package org.elasticsearch.action.admin.indices.exists.types;
import com.google.common.collect.ImmutableMap;
import org.elasticsearch.ElasticSearchException; import org.elasticsearch.ElasticSearchException;
import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.support.master.TransportMasterNodeOperationAction; import org.elasticsearch.action.support.master.TransportMasterNodeOperationAction;
@ -33,6 +32,8 @@ import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.TransportService; import org.elasticsearch.transport.TransportService;
import java.util.Map;
/** /**
* Types exists transport action. * Types exists transport action.
*/ */
@ -84,7 +85,7 @@ public class TransportTypesExistsAction extends TransportMasterNodeOperationActi
return; return;
} }
ImmutableMap<String, MappingMetaData> mappings = state.metaData().getIndices().get(concreteIndex).mappings(); Map<String, MappingMetaData> mappings = state.metaData().getIndices().get(concreteIndex).mappings();
if (mappings.isEmpty()) { if (mappings.isEmpty()) {
listener.onResponse(new TypesExistsResponse(false)); listener.onResponse(new TypesExistsResponse(false));
return; return;

View File

@ -20,7 +20,6 @@
package org.elasticsearch.cluster; package org.elasticsearch.cluster;
import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists; import com.google.common.collect.Lists;
import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.metadata.MetaData;
@ -166,8 +165,8 @@ public class ClusterChangedEvent {
public boolean indicesStateChanged() { public boolean indicesStateChanged() {
if (metaDataChanged()) { if (metaDataChanged()) {
ImmutableMap<String,IndexMetaData> indices = state.metaData().indices(); Map<String,IndexMetaData> indices = state.metaData().indices();
ImmutableMap<String,IndexMetaData> previousIndices = previousState.metaData().indices(); Map<String,IndexMetaData> previousIndices = previousState.metaData().indices();
for (Map.Entry<String, IndexMetaData> entry : indices.entrySet()) { for (Map.Entry<String, IndexMetaData> entry : indices.entrySet()) {
IndexMetaData indexMetaData = entry.getValue(); IndexMetaData indexMetaData = entry.getValue();

View File

@ -19,6 +19,7 @@
package org.elasticsearch.cluster.metadata; package org.elasticsearch.cluster.metadata;
import com.google.common.collect.ImmutableSet;
import org.elasticsearch.ElasticSearchGenerationException; import org.elasticsearch.ElasticSearchGenerationException;
import org.elasticsearch.common.Strings; import org.elasticsearch.common.Strings;
import org.elasticsearch.common.compress.CompressedString; import org.elasticsearch.common.compress.CompressedString;
@ -30,7 +31,9 @@ import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentParser;
import java.io.IOException; import java.io.IOException;
import java.util.Collections;
import java.util.Map; import java.util.Map;
import java.util.Set;
/** /**
* *
@ -41,15 +44,22 @@ public class AliasMetaData {
private final CompressedString filter; private final CompressedString filter;
private String indexRouting; private final String indexRouting;
private String searchRouting; private final String searchRouting;
private final Set<String> searchRoutingValues;
private AliasMetaData(String alias, CompressedString filter, String indexRouting, String searchRouting) { private AliasMetaData(String alias, CompressedString filter, String indexRouting, String searchRouting) {
this.alias = alias; this.alias = alias;
this.filter = filter; this.filter = filter;
this.indexRouting = indexRouting; this.indexRouting = indexRouting;
this.searchRouting = searchRouting; this.searchRouting = searchRouting;
if (searchRouting != null) {
searchRoutingValues = Collections.unmodifiableSet(Strings.splitStringByCommaToSet(searchRouting));
} else {
searchRoutingValues = ImmutableSet.of();
}
} }
public String alias() { public String alias() {
@ -68,6 +78,10 @@ public class AliasMetaData {
return filter(); return filter();
} }
public boolean filteringRequired() {
return filter != null;
}
public String getSearchRouting() { public String getSearchRouting() {
return searchRouting(); return searchRouting();
} }
@ -84,6 +98,10 @@ public class AliasMetaData {
return indexRouting; return indexRouting;
} }
public Set<String> searchRoutingValues() {
return searchRoutingValues;
}
public static Builder builder(String alias) { public static Builder builder(String alias) {
return new Builder(alias); return new Builder(alias);
} }

View File

@ -161,13 +161,13 @@ public class IndexMetaData {
private final State state; private final State state;
private final ImmutableMap<String, AliasMetaData> aliases; private final Map<String, AliasMetaData> aliases;
private final Settings settings; private final Settings settings;
private final ImmutableMap<String, MappingMetaData> mappings; private final Map<String, MappingMetaData> mappings;
private final ImmutableMap<String, Custom> customs; private final Map<String, Custom> customs;
private transient final int totalNumberOfShards; private transient final int totalNumberOfShards;
@ -175,7 +175,7 @@ public class IndexMetaData {
private final DiscoveryNodeFilters includeFilters; private final DiscoveryNodeFilters includeFilters;
private final DiscoveryNodeFilters excludeFilters; private final DiscoveryNodeFilters excludeFilters;
private IndexMetaData(String index, long version, State state, Settings settings, ImmutableMap<String, MappingMetaData> mappings, ImmutableMap<String, AliasMetaData> aliases, ImmutableMap<String, Custom> customs) { private IndexMetaData(String index, long version, State state, Settings settings, Map<String, MappingMetaData> mappings, Map<String, AliasMetaData> aliases, Map<String, Custom> customs) {
Preconditions.checkArgument(settings.getAsInt(SETTING_NUMBER_OF_SHARDS, -1) != -1, "must specify numberOfShards for index [" + index + "]"); Preconditions.checkArgument(settings.getAsInt(SETTING_NUMBER_OF_SHARDS, -1) != -1, "must specify numberOfShards for index [" + index + "]");
Preconditions.checkArgument(settings.getAsInt(SETTING_NUMBER_OF_REPLICAS, -1) != -1, "must specify numberOfReplicas for index [" + index + "]"); Preconditions.checkArgument(settings.getAsInt(SETTING_NUMBER_OF_REPLICAS, -1) != -1, "must specify numberOfReplicas for index [" + index + "]");
this.index = index; this.index = index;
@ -264,19 +264,19 @@ public class IndexMetaData {
return settings(); return settings();
} }
public ImmutableMap<String, AliasMetaData> aliases() { public Map<String, AliasMetaData> aliases() {
return this.aliases; return this.aliases;
} }
public ImmutableMap<String, AliasMetaData> getAliases() { public Map<String, AliasMetaData> getAliases() {
return aliases(); return aliases();
} }
public ImmutableMap<String, MappingMetaData> mappings() { public Map<String, MappingMetaData> mappings() {
return mappings; return mappings;
} }
public ImmutableMap<String, MappingMetaData> getMappings() { public Map<String, MappingMetaData> getMappings() {
return mappings(); return mappings();
} }
@ -301,11 +301,11 @@ public class IndexMetaData {
return mappings.get(MapperService.DEFAULT_MAPPING); return mappings.get(MapperService.DEFAULT_MAPPING);
} }
public ImmutableMap<String, Custom> customs() { public Map<String, Custom> customs() {
return this.customs; return this.customs;
} }
public ImmutableMap<String, Custom> getCustoms() { public Map<String, Custom> getCustoms() {
return this.customs; return this.customs;
} }
@ -508,7 +508,7 @@ public class IndexMetaData {
AliasMetaData aliasMd = AliasMetaData.newAliasMetaDataBuilder(alias).build(); AliasMetaData aliasMd = AliasMetaData.newAliasMetaDataBuilder(alias).build();
tmpAliases.put(alias, aliasMd); tmpAliases.put(alias, aliasMd);
} }
tmpAliases.putAll(aliases.immutableMap()); tmpAliases.putAll(aliases.map());
// Remove index.aliases from settings once they are migrated to the new data structure // Remove index.aliases from settings once they are migrated to the new data structure
tmpSettings = ImmutableSettings.settingsBuilder().put(settings).putArray("index.aliases").build(); tmpSettings = ImmutableSettings.settingsBuilder().put(settings).putArray("index.aliases").build();
} }
@ -521,7 +521,7 @@ public class IndexMetaData {
} }
} }
return new IndexMetaData(index, version, state, tmpSettings, mappings.immutableMap(), tmpAliases.immutableMap(), customs.immutableMap()); return new IndexMetaData(index, version, state, tmpSettings, mappings.readOnlyMap(), tmpAliases.readOnlyMap(), customs.readOnlyMap());
} }
public static void toXContent(IndexMetaData indexMetaData, XContentBuilder builder, ToXContent.Params params) throws IOException { public static void toXContent(IndexMetaData indexMetaData, XContentBuilder builder, ToXContent.Params params) throws IOException {

View File

@ -22,6 +22,8 @@ package org.elasticsearch.cluster.metadata;
import com.google.common.base.Predicate; import com.google.common.base.Predicate;
import com.google.common.collect.*; import com.google.common.collect.*;
import gnu.trove.set.hash.THashSet; import gnu.trove.set.hash.THashSet;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.RamUsageEstimator;
import org.elasticsearch.ElasticSearchIllegalArgumentException; import org.elasticsearch.ElasticSearchIllegalArgumentException;
import org.elasticsearch.action.support.IgnoreIndices; import org.elasticsearch.action.support.IgnoreIndices;
import org.elasticsearch.cluster.block.ClusterBlock; import org.elasticsearch.cluster.block.ClusterBlock;
@ -29,12 +31,14 @@ import org.elasticsearch.cluster.block.ClusterBlockLevel;
import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.Strings; import org.elasticsearch.common.Strings;
import org.elasticsearch.common.collect.MapBuilder; import org.elasticsearch.common.collect.MapBuilder;
import org.elasticsearch.common.collect.XMaps;
import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.regex.Regex;
import org.elasticsearch.common.settings.ImmutableSettings; import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.settings.loader.SettingsLoader; import org.elasticsearch.common.settings.loader.SettingsLoader;
import org.elasticsearch.common.trove.ExtTHashMap;
import org.elasticsearch.common.xcontent.*; import org.elasticsearch.common.xcontent.*;
import org.elasticsearch.index.Index; import org.elasticsearch.index.Index;
import org.elasticsearch.indices.IndexMissingException; import org.elasticsearch.indices.IndexMissingException;
@ -46,7 +50,6 @@ import java.util.*;
import static com.google.common.collect.Lists.newArrayList; import static com.google.common.collect.Lists.newArrayList;
import static com.google.common.collect.Maps.newHashMap; import static com.google.common.collect.Maps.newHashMap;
import static com.google.common.collect.Sets.newHashSet;
import static org.elasticsearch.common.collect.MapBuilder.newMapBuilder; import static org.elasticsearch.common.collect.MapBuilder.newMapBuilder;
import static org.elasticsearch.common.settings.ImmutableSettings.*; import static org.elasticsearch.common.settings.ImmutableSettings.*;
@ -105,9 +108,9 @@ public class MetaData implements Iterable<IndexMetaData> {
private final Settings transientSettings; private final Settings transientSettings;
private final Settings persistentSettings; private final Settings persistentSettings;
private final Settings settings; private final Settings settings;
private final ImmutableMap<String, IndexMetaData> indices; private final Map<String, IndexMetaData> indices;
private final ImmutableMap<String, IndexTemplateMetaData> templates; private final Map<String, IndexTemplateMetaData> templates;
private final ImmutableMap<String, Custom> customs; private final Map<String, Custom> customs;
private final transient int totalNumberOfShards; // Transient ? not serializable anyway? private final transient int totalNumberOfShards; // Transient ? not serializable anyway?
private final int numberOfShards; private final int numberOfShards;
@ -117,29 +120,24 @@ public class MetaData implements Iterable<IndexMetaData> {
private final ImmutableSet<String> allIndicesSet; private final ImmutableSet<String> allIndicesSet;
private final String[] allOpenIndices; private final String[] allOpenIndices;
private final ImmutableMap<String, ImmutableMap<String, AliasMetaData>> aliases; private final Map<String, Map<String, AliasMetaData>> aliases;
private final Map<String, StringArray> aliasAndIndexToIndexMap;
private final ImmutableMap<String, ImmutableMap<String, ImmutableSet<String>>> aliasToIndexToSearchRoutingMap; MetaData(long version, Settings transientSettings, Settings persistentSettings, Map<String, IndexMetaData> indices, Map<String, IndexTemplateMetaData> templates, Map<String, Custom> customs) {
// This map indicates if an alias associated with an index is filtering alias
private final ImmutableMap<String, ImmutableMap<String, Boolean>> indexToAliasFilteringRequiredMap;
private final ImmutableMap<String, String[]> aliasAndIndexToIndexMap;
MetaData(long version, Settings transientSettings, Settings persistentSettings, ImmutableMap<String, IndexMetaData> indices, ImmutableMap<String, IndexTemplateMetaData> templates, ImmutableMap<String, Custom> customs) {
this.version = version; this.version = version;
this.transientSettings = transientSettings; this.transientSettings = transientSettings;
this.persistentSettings = persistentSettings; this.persistentSettings = persistentSettings;
this.settings = ImmutableSettings.settingsBuilder().put(persistentSettings).put(transientSettings).build(); this.settings = ImmutableSettings.settingsBuilder().put(persistentSettings).put(transientSettings).build();
this.indices = ImmutableMap.copyOf(indices); this.indices = indices;
this.customs = customs; this.customs = customs;
this.templates = templates; this.templates = templates;
int totalNumberOfShards = 0; int totalNumberOfShards = 0;
int numberOfShards = 0; int numberOfShards = 0;
int numAliases = 0;
for (IndexMetaData indexMetaData : indices.values()) { for (IndexMetaData indexMetaData : indices.values()) {
totalNumberOfShards += indexMetaData.totalNumberOfShards(); totalNumberOfShards += indexMetaData.totalNumberOfShards();
numberOfShards += indexMetaData.numberOfShards(); numberOfShards += indexMetaData.numberOfShards();
numAliases += indexMetaData.aliases().size();
} }
this.totalNumberOfShards = totalNumberOfShards; this.totalNumberOfShards = totalNumberOfShards;
this.numberOfShards = numberOfShards; this.numberOfShards = numberOfShards;
@ -151,6 +149,7 @@ public class MetaData implements Iterable<IndexMetaData> {
} }
allIndices = allIndicesLst.toArray(new String[allIndicesLst.size()]); allIndices = allIndicesLst.toArray(new String[allIndicesLst.size()]);
allIndicesSet = ImmutableSet.copyOf(allIndices); allIndicesSet = ImmutableSet.copyOf(allIndices);
int numIndices = allIndicesSet.size();
List<String> allOpenIndices = Lists.newArrayList(); List<String> allOpenIndices = Lists.newArrayList();
for (IndexMetaData indexMetaData : indices.values()) { for (IndexMetaData indexMetaData : indices.values()) {
@ -161,88 +160,49 @@ public class MetaData implements Iterable<IndexMetaData> {
this.allOpenIndices = allOpenIndices.toArray(new String[allOpenIndices.size()]); this.allOpenIndices = allOpenIndices.toArray(new String[allOpenIndices.size()]);
// build aliases map // build aliases map
MapBuilder<String, MapBuilder<String, AliasMetaData>> tmpAliasesMap = newMapBuilder(); ExtTHashMap<String, Map<String, AliasMetaData>> aliases = new ExtTHashMap<String, Map<String, AliasMetaData>>(numAliases);
for (IndexMetaData indexMetaData : indices.values()) { for (IndexMetaData indexMetaData : indices.values()) {
String index = indexMetaData.index(); String index = indexMetaData.index();
for (AliasMetaData aliasMd : indexMetaData.aliases().values()) { for (AliasMetaData aliasMd : indexMetaData.aliases().values()) {
MapBuilder<String, AliasMetaData> indexAliasMap = tmpAliasesMap.get(aliasMd.alias()); Map<String, AliasMetaData> indexAliasMap = aliases.get(aliasMd.alias());
if (indexAliasMap == null) { if (indexAliasMap == null) {
indexAliasMap = newMapBuilder(); indexAliasMap = new ExtTHashMap<String, AliasMetaData>(indices.size());
tmpAliasesMap.put(aliasMd.alias(), indexAliasMap); aliases.put(aliasMd.alias(), indexAliasMap);
} }
indexAliasMap.put(index, aliasMd); indexAliasMap.put(index, aliasMd);
} }
} }
MapBuilder<String, ImmutableMap<String, AliasMetaData>> aliases = newMapBuilder(); for (int i = 0; i < aliases.internalValues().length; i++) {
for (Map.Entry<String, MapBuilder<String, AliasMetaData>> alias : tmpAliasesMap.map().entrySet()) { if (aliases.internalValues()[i] != null) {
aliases.put(alias.getKey(), alias.getValue().immutableMap()); aliases.internalValues()[i] = XMaps.makeReadOnly((Map) aliases.internalValues()[i]);
} }
this.aliases = aliases.immutableMap(); }
this.aliases = XMaps.makeReadOnly(aliases);
// build routing aliases set ExtTHashMap<String, StringArray> aliasAndIndexToIndexMap = new ExtTHashMap<String, StringArray>(numAliases + numIndices);
MapBuilder<String, MapBuilder<String, ImmutableSet<String>>> tmpAliasToIndexToSearchRoutingMap = newMapBuilder();
for (IndexMetaData indexMetaData : indices.values()) { for (IndexMetaData indexMetaData : indices.values()) {
for (AliasMetaData aliasMd : indexMetaData.aliases().values()) { StringArray indicesLst = aliasAndIndexToIndexMap.get(indexMetaData.index());
MapBuilder<String, ImmutableSet<String>> indexToSearchRoutingMap = tmpAliasToIndexToSearchRoutingMap.get(aliasMd.alias()); if (indicesLst == null) {
if (indexToSearchRoutingMap == null) { indicesLst = new StringArray();
indexToSearchRoutingMap = newMapBuilder(); aliasAndIndexToIndexMap.put(indexMetaData.index(), indicesLst);
tmpAliasToIndexToSearchRoutingMap.put(aliasMd.alias(), indexToSearchRoutingMap);
} }
if (aliasMd.searchRouting() != null) { indicesLst.add(indexMetaData.index());
indexToSearchRoutingMap.put(indexMetaData.index(), ImmutableSet.copyOf(Strings.splitStringByCommaToSet(aliasMd.searchRouting())));
} else {
indexToSearchRoutingMap.put(indexMetaData.index(), ImmutableSet.<String>of());
}
}
}
MapBuilder<String, ImmutableMap<String, ImmutableSet<String>>> aliasToIndexToSearchRoutingMap = newMapBuilder();
for (Map.Entry<String, MapBuilder<String, ImmutableSet<String>>> alias : tmpAliasToIndexToSearchRoutingMap.map().entrySet()) {
aliasToIndexToSearchRoutingMap.put(alias.getKey(), alias.getValue().immutableMap());
}
this.aliasToIndexToSearchRoutingMap = aliasToIndexToSearchRoutingMap.immutableMap();
// build filtering required map
MapBuilder<String, ImmutableMap<String, Boolean>> filteringRequiredMap = newMapBuilder();
for (IndexMetaData indexMetaData : indices.values()) {
MapBuilder<String, Boolean> indexFilteringRequiredMap = newMapBuilder();
// Filtering is not required for the index itself
indexFilteringRequiredMap.put(indexMetaData.index(), false);
for (AliasMetaData aliasMetaData : indexMetaData.aliases().values()) {
if (aliasMetaData.filter() != null) {
indexFilteringRequiredMap.put(aliasMetaData.alias(), true);
} else {
indexFilteringRequiredMap.put(aliasMetaData.alias(), false);
}
}
filteringRequiredMap.put(indexMetaData.index(), indexFilteringRequiredMap.immutableMap());
}
indexToAliasFilteringRequiredMap = filteringRequiredMap.immutableMap();
// build aliasAndIndex to Index map
MapBuilder<String, Set<String>> tmpAliasAndIndexToIndexBuilder = newMapBuilder();
for (IndexMetaData indexMetaData : indices.values()) {
Set<String> lst = tmpAliasAndIndexToIndexBuilder.get(indexMetaData.index());
if (lst == null) {
lst = newHashSet();
tmpAliasAndIndexToIndexBuilder.put(indexMetaData.index(), lst);
}
lst.add(indexMetaData.index());
for (String alias : indexMetaData.aliases().keySet()) { for (String alias : indexMetaData.aliases().keySet()) {
lst = tmpAliasAndIndexToIndexBuilder.get(alias); indicesLst = aliasAndIndexToIndexMap.get(alias);
if (lst == null) { if (indicesLst == null) {
lst = newHashSet(); indicesLst = new StringArray();
tmpAliasAndIndexToIndexBuilder.put(alias, lst); aliasAndIndexToIndexMap.put(alias, indicesLst);
} }
lst.add(indexMetaData.index()); indicesLst.add(indexMetaData.index());
} }
} }
MapBuilder<String, String[]> aliasAndIndexToIndexBuilder = newMapBuilder(); for (StringArray stringArray : aliasAndIndexToIndexMap.values()) {
for (Map.Entry<String, Set<String>> entry : tmpAliasAndIndexToIndexBuilder.map().entrySet()) { stringArray.trim();
aliasAndIndexToIndexBuilder.put(entry.getKey(), entry.getValue().toArray(new String[entry.getValue().size()]));
} }
this.aliasAndIndexToIndexMap = aliasAndIndexToIndexBuilder.immutableMap();
this.aliasAndIndexToIndexMap = XMaps.makeReadOnly(aliasAndIndexToIndexMap);
} }
public long version() { public long version() {
@ -264,11 +224,11 @@ public class MetaData implements Iterable<IndexMetaData> {
return this.persistentSettings; return this.persistentSettings;
} }
public ImmutableMap<String, ImmutableMap<String, AliasMetaData>> aliases() { public Map<String, Map<String, AliasMetaData>> aliases() {
return this.aliases; return this.aliases;
} }
public ImmutableMap<String, ImmutableMap<String, AliasMetaData>> getAliases() { public Map<String, Map<String, AliasMetaData>> getAliases() {
return aliases(); return aliases();
} }
@ -438,7 +398,7 @@ public class MetaData implements Iterable<IndexMetaData> {
*/ */
public String resolveIndexRouting(@Nullable String routing, String aliasOrIndex) { public String resolveIndexRouting(@Nullable String routing, String aliasOrIndex) {
// Check if index is specified by an alias // Check if index is specified by an alias
ImmutableMap<String, AliasMetaData> indexAliases = aliases.get(aliasOrIndex); Map<String, AliasMetaData> indexAliases = aliases.get(aliasOrIndex);
if (indexAliases == null || indexAliases.isEmpty()) { if (indexAliases == null || indexAliases.isEmpty()) {
return routing; return routing;
} }
@ -486,11 +446,11 @@ public class MetaData implements Iterable<IndexMetaData> {
} }
for (String aliasOrIndex : aliasesOrIndices) { for (String aliasOrIndex : aliasesOrIndices) {
ImmutableMap<String, ImmutableSet<String>> indexToRoutingMap = aliasToIndexToSearchRoutingMap.get(aliasOrIndex); Map<String, AliasMetaData> indexToRoutingMap = aliases.get(aliasOrIndex);
if (indexToRoutingMap != null && !indexToRoutingMap.isEmpty()) { if (indexToRoutingMap != null && !indexToRoutingMap.isEmpty()) {
for (Map.Entry<String, ImmutableSet<String>> indexRouting : indexToRoutingMap.entrySet()) { for (Map.Entry<String, AliasMetaData> indexRouting : indexToRoutingMap.entrySet()) {
if (!norouting.contains(indexRouting.getKey())) { if (!norouting.contains(indexRouting.getKey())) {
if (!indexRouting.getValue().isEmpty()) { if (!indexRouting.getValue().searchRoutingValues().isEmpty()) {
// Routing alias // Routing alias
if (routings == null) { if (routings == null) {
routings = newHashMap(); routings = newHashMap();
@ -500,7 +460,7 @@ public class MetaData implements Iterable<IndexMetaData> {
r = new THashSet<String>(); r = new THashSet<String>();
routings.put(indexRouting.getKey(), r); routings.put(indexRouting.getKey(), r);
} }
r.addAll(indexRouting.getValue()); r.addAll(indexRouting.getValue().searchRoutingValues());
if (paramRouting != null) { if (paramRouting != null) {
r.retainAll(paramRouting); r.retainAll(paramRouting);
} }
@ -558,13 +518,13 @@ public class MetaData implements Iterable<IndexMetaData> {
paramRouting = Strings.splitStringByCommaToSet(routing); paramRouting = Strings.splitStringByCommaToSet(routing);
} }
ImmutableMap<String, ImmutableSet<String>> indexToRoutingMap = aliasToIndexToSearchRoutingMap.get(aliasOrIndex); Map<String, AliasMetaData> indexToRoutingMap = aliases.get(aliasOrIndex);
if (indexToRoutingMap != null && !indexToRoutingMap.isEmpty()) { if (indexToRoutingMap != null && !indexToRoutingMap.isEmpty()) {
// It's an alias // It's an alias
for (Map.Entry<String, ImmutableSet<String>> indexRouting : indexToRoutingMap.entrySet()) { for (Map.Entry<String, AliasMetaData> indexRouting : indexToRoutingMap.entrySet()) {
if (!indexRouting.getValue().isEmpty()) { if (!indexRouting.getValue().searchRoutingValues().isEmpty()) {
// Routing alias // Routing alias
Set<String> r = new THashSet<String>(indexRouting.getValue()); Set<String> r = new THashSet<String>(indexRouting.getValue().searchRoutingValues());
if (paramRouting != null) { if (paramRouting != null) {
r.retainAll(paramRouting); r.retainAll(paramRouting);
} }
@ -639,11 +599,11 @@ public class MetaData implements Iterable<IndexMetaData> {
if (this.indices.containsKey(aliasOrIndex)) { if (this.indices.containsKey(aliasOrIndex)) {
return aliasesOrIndices; return aliasesOrIndices;
} }
String[] actualLst = aliasAndIndexToIndexMap.get(aliasOrIndex); StringArray actualLst = aliasAndIndexToIndexMap.get(aliasOrIndex);
if (actualLst == null) { if (actualLst == null) {
throw new IndexMissingException(new Index(aliasOrIndex)); throw new IndexMissingException(new Index(aliasOrIndex));
} else { } else {
return actualLst; return actualLst.values;
} }
} }
@ -662,13 +622,13 @@ public class MetaData implements Iterable<IndexMetaData> {
Set<String> actualIndices = new THashSet<String>(); Set<String> actualIndices = new THashSet<String>();
for (String index : aliasesOrIndices) { for (String index : aliasesOrIndices) {
String[] actualLst = aliasAndIndexToIndexMap.get(index); StringArray actualLst = aliasAndIndexToIndexMap.get(index);
if (actualLst == null) { if (actualLst == null) {
if (ignoreIndices != IgnoreIndices.MISSING) { if (ignoreIndices != IgnoreIndices.MISSING) {
throw new IndexMissingException(new Index(index)); throw new IndexMissingException(new Index(index));
} }
} else { } else {
for (String x : actualLst) { for (String x : actualLst.values) {
actualIndices.add(x); actualIndices.add(x);
} }
} }
@ -686,14 +646,14 @@ public class MetaData implements Iterable<IndexMetaData> {
return index; return index;
} }
// not an actual index, fetch from an alias // not an actual index, fetch from an alias
String[] lst = aliasAndIndexToIndexMap.get(index); StringArray lst = aliasAndIndexToIndexMap.get(index);
if (lst == null) { if (lst == null) {
throw new IndexMissingException(new Index(index)); throw new IndexMissingException(new Index(index));
} }
if (lst.length > 1) { if (lst.values.length > 1) {
throw new ElasticSearchIllegalArgumentException("Alias [" + index + "] has more than one indices associated with it [" + Arrays.toString(lst) + "], can't execute a single index op"); throw new ElasticSearchIllegalArgumentException("Alias [" + index + "] has more than one indices associated with it [" + Arrays.toString(lst.values) + "], can't execute a single index op");
} }
return lst[0]; return lst.values[0];
} }
public String[] convertFromWildcards(String[] aliasesOrIndices, boolean wildcardOnlyOpen, IgnoreIndices ignoreIndices) { public String[] convertFromWildcards(String[] aliasesOrIndices, boolean wildcardOnlyOpen, IgnoreIndices ignoreIndices) {
@ -780,27 +740,27 @@ public class MetaData implements Iterable<IndexMetaData> {
return indices.get(index); return indices.get(index);
} }
public ImmutableMap<String, IndexMetaData> indices() { public Map<String, IndexMetaData> indices() {
return this.indices; return this.indices;
} }
public ImmutableMap<String, IndexMetaData> getIndices() { public Map<String, IndexMetaData> getIndices() {
return indices(); return indices();
} }
public ImmutableMap<String, IndexTemplateMetaData> templates() { public Map<String, IndexTemplateMetaData> templates() {
return this.templates; return this.templates;
} }
public ImmutableMap<String, IndexTemplateMetaData> getTemplates() { public Map<String, IndexTemplateMetaData> getTemplates() {
return this.templates; return this.templates;
} }
public ImmutableMap<String, Custom> customs() { public Map<String, Custom> customs() {
return this.customs; return this.customs;
} }
public ImmutableMap<String, Custom> getCustoms() { public Map<String, Custom> getCustoms() {
return this.customs; return this.customs;
} }
@ -828,35 +788,42 @@ public class MetaData implements Iterable<IndexMetaData> {
* <p>Only aliases with filters are returned. If the indices list contains a non-filtering reference to * <p>Only aliases with filters are returned. If the indices list contains a non-filtering reference to
* the index itself - null is returned. Returns <tt>null</tt> if no filtering is required.</p> * the index itself - null is returned. Returns <tt>null</tt> if no filtering is required.</p>
*/ */
public String[] filteringAliases(String index, String... indices) { public String[] filteringAliases(String index, String... indicesOrAliases) {
if (isAllIndices(indices)) { if (isAllIndices(indicesOrAliases)) {
return null; return null;
} }
// optimize for the most common single index/alias scenario // optimize for the most common single index/alias scenario
if (indices.length == 1) { if (indicesOrAliases.length == 1) {
String alias = indices[0]; String alias = indicesOrAliases[0];
ImmutableMap<String, Boolean> aliasToFilteringRequiredMap = indexToAliasFilteringRequiredMap.get(index); IndexMetaData indexMetaData = this.indices.get(index);
if (aliasToFilteringRequiredMap == null) { if (indexMetaData == null) {
// Shouldn't happen // Shouldn't happen
throw new IndexMissingException(new Index(index)); throw new IndexMissingException(new Index(index));
} }
Boolean filteringRequired = aliasToFilteringRequiredMap.get(alias); AliasMetaData aliasMetaData = indexMetaData.aliases().get(alias);
if (filteringRequired == null || !filteringRequired) { boolean filteringRequired = aliasMetaData != null && aliasMetaData.filteringRequired();
if (!filteringRequired) {
return null; return null;
} }
return new String[]{alias}; return new String[]{alias};
} }
List<String> filteringAliases = null; List<String> filteringAliases = null;
for (String alias : indices) { for (String alias : indicesOrAliases) {
ImmutableMap<String, Boolean> aliasToFilteringRequiredMap = indexToAliasFilteringRequiredMap.get(index); if (alias.equals(index)) {
if (aliasToFilteringRequiredMap == null) { return null;
}
IndexMetaData indexMetaData = this.indices.get(index);
if (indexMetaData == null) {
// Shouldn't happen // Shouldn't happen
throw new IndexMissingException(new Index(index)); throw new IndexMissingException(new Index(index));
} }
Boolean filteringRequired = aliasToFilteringRequiredMap.get(alias);
AliasMetaData aliasMetaData = indexMetaData.aliases().get(alias);
// Check that this is an alias for the current index // Check that this is an alias for the current index
// Otherwise - skip it // Otherwise - skip it
if (filteringRequired != null) { if (aliasMetaData != null) {
boolean filteringRequired = aliasMetaData.filteringRequired();
if (filteringRequired) { if (filteringRequired) {
// If filtering required - add it to the list of filters // If filtering required - add it to the list of filters
if (filteringAliases == null) { if (filteringAliases == null) {
@ -925,7 +892,7 @@ public class MetaData implements Iterable<IndexMetaData> {
} }
@Override @Override
public UnmodifiableIterator<IndexMetaData> iterator() { public Iterator<IndexMetaData> iterator() {
return indices.values().iterator(); return indices.values().iterator();
} }
@ -1082,7 +1049,7 @@ public class MetaData implements Iterable<IndexMetaData> {
} }
public MetaData build() { public MetaData build() {
return new MetaData(version, transientSettings, persistentSettings, indices.immutableMap(), templates.immutableMap(), customs.immutableMap()); return new MetaData(version, transientSettings, persistentSettings, indices.readOnlyMap(), templates.readOnlyMap(), customs.readOnlyMap());
} }
public static String toXContent(MetaData metaData) throws IOException { public static String toXContent(MetaData metaData) throws IOException {
@ -1224,4 +1191,35 @@ public class MetaData implements Iterable<IndexMetaData> {
} }
} }
} }
static class StringArray {
String[] values = new String[1];
int head = 0;
void add(String value) {
if (head == values.length) {
grow();
}
values[head++] = value;
}
void grow() {
int newSize = values.length + 1;
String[] newValues = new String[ArrayUtil.oversize(newSize, RamUsageEstimator.NUM_BYTES_OBJECT_REF)];
System.arraycopy(values, 0, newValues, 0, values.length);
values = newValues;
}
void trim() {
if (values.length == head) {
return;
}
String[] newValues = new String[head];
System.arraycopy(values, 0, newValues, 0, head);
values = newValues;
}
}
} }

View File

@ -110,6 +110,7 @@ public class MetaDataIndexAliasesService extends AbstractComponent {
if (indexMetaData == null) { if (indexMetaData == null) {
throw new IndexMissingException(new Index(aliasAction.index())); throw new IndexMissingException(new Index(aliasAction.index()));
} }
// TODO: not copy (putAll)
IndexMetaData.Builder indexMetaDataBuilder = newIndexMetaDataBuilder(indexMetaData); IndexMetaData.Builder indexMetaDataBuilder = newIndexMetaDataBuilder(indexMetaData);
if (aliasAction.actionType() == AliasAction.Type.ADD) { if (aliasAction.actionType() == AliasAction.Type.ADD) {
String filter = aliasAction.filter(); String filter = aliasAction.filter();

View File

@ -80,6 +80,10 @@ public class MapBuilder<K, V> {
return this.map; return this.map;
} }
public Map<K, V> readOnlyMap() {
return XMaps.makeReadOnly(map);
}
public ImmutableMap<K, V> immutableMap() { public ImmutableMap<K, V> immutableMap() {
return ImmutableMap.copyOf(map); return ImmutableMap.copyOf(map);
} }

View File

@ -19,14 +19,13 @@
package org.elasticsearch.common.collect; package org.elasticsearch.common.collect;
import com.google.common.collect.ForwardingMap;
import gnu.trove.impl.Constants; import gnu.trove.impl.Constants;
import java.util.Map;
import org.elasticsearch.ElasticSearchIllegalArgumentException; import org.elasticsearch.ElasticSearchIllegalArgumentException;
import org.elasticsearch.common.trove.ExtTHashMap; import org.elasticsearch.common.trove.ExtTHashMap;
import com.google.common.collect.ForwardingMap; import java.util.Collections;
import java.util.Map;
/** /**
* This class provides factory methods for Maps. The returned {@link Map} * This class provides factory methods for Maps. The returned {@link Map}
@ -89,4 +88,12 @@ public final class XMaps {
} }
}; };
} }
/**
* Wraps the given map into a read only implementation.
*/
public static <K, V> Map<K, V> makeReadOnly(Map<K, V> map) {
return Collections.unmodifiableMap(map);
}
} }

View File

@ -19,8 +19,6 @@
package org.elasticsearch.index.aliases; package org.elasticsearch.index.aliases;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.UnmodifiableIterator;
import org.apache.lucene.queries.FilterClause; import org.apache.lucene.queries.FilterClause;
import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.Filter; import org.apache.lucene.search.Filter;
@ -29,6 +27,7 @@ import org.elasticsearch.common.compress.CompressedString;
import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.lucene.search.XBooleanFilter; import org.elasticsearch.common.lucene.search.XBooleanFilter;
import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.AbstractIndexComponent; import org.elasticsearch.index.AbstractIndexComponent;
@ -39,20 +38,16 @@ import org.elasticsearch.indices.AliasFilterParsingException;
import org.elasticsearch.indices.InvalidAliasNameException; import org.elasticsearch.indices.InvalidAliasNameException;
import java.io.IOException; import java.io.IOException;
import java.util.Iterator;
import java.util.Map; import java.util.Map;
import static org.elasticsearch.common.collect.MapBuilder.newMapBuilder;
/** /**
* *
*/ */
public class IndexAliasesService extends AbstractIndexComponent implements Iterable<IndexAlias> { public class IndexAliasesService extends AbstractIndexComponent implements Iterable<IndexAlias> {
private final IndexQueryParserService indexQueryParser; private final IndexQueryParserService indexQueryParser;
private final Map<String, IndexAlias> aliases = ConcurrentCollections.newConcurrentMapWithAggressiveConcurrency();
private volatile ImmutableMap<String, IndexAlias> aliases = ImmutableMap.of();
private final Object mutex = new Object();
@Inject @Inject
public IndexAliasesService(Index index, @IndexSettings Settings indexSettings, IndexQueryParserService indexQueryParser) { public IndexAliasesService(Index index, @IndexSettings Settings indexSettings, IndexQueryParserService indexQueryParser) {
@ -77,9 +72,7 @@ public class IndexAliasesService extends AbstractIndexComponent implements Itera
} }
public void addAll(Map<String, IndexAlias> aliases) { public void addAll(Map<String, IndexAlias> aliases) {
synchronized (mutex) { this.aliases.putAll(aliases);
this.aliases = newMapBuilder(this.aliases).putAll(aliases).immutableMap();
}
} }
/** /**
@ -126,15 +119,11 @@ public class IndexAliasesService extends AbstractIndexComponent implements Itera
} }
private void add(IndexAlias indexAlias) { private void add(IndexAlias indexAlias) {
synchronized (mutex) { aliases.put(indexAlias.alias(), indexAlias);
aliases = newMapBuilder(aliases).put(indexAlias.alias(), indexAlias).immutableMap();
}
} }
public void remove(String alias) { public void remove(String alias) {
synchronized (mutex) { aliases.remove(alias);
aliases = newMapBuilder(aliases).remove(alias).immutableMap();
}
} }
private Filter parse(String alias, CompressedString filter) { private Filter parse(String alias, CompressedString filter) {
@ -155,7 +144,7 @@ public class IndexAliasesService extends AbstractIndexComponent implements Itera
} }
@Override @Override
public UnmodifiableIterator<IndexAlias> iterator() { public Iterator<IndexAlias> iterator() {
return aliases.values().iterator(); return aliases.values().iterator();
} }
} }

View File

@ -0,0 +1,133 @@
/*
* Licensed to Elastic Search and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. Elastic Search licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.benchmark.aliases;
import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder;
import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse;
import org.elasticsearch.client.Client;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.indices.AliasMissingException;
import org.elasticsearch.indices.IndexAlreadyExistsException;
import org.elasticsearch.node.Node;
import org.elasticsearch.node.NodeBuilder;
import java.io.IOException;
/**
*/
public class AliasesBenchmark {
private final static String INDEX_NAME = "my-index";
public static void main(String[] args) throws IOException {
int NUM_ADDITIONAL_NODES = 0;
int BASE_ALIAS_COUNT = 100000;
int NUM_ADD_ALIAS_REQUEST = 1000;
Settings settings = ImmutableSettings.settingsBuilder()
.put("gateway.type", "local")
.put("node.master", false).build();
Node node1 = NodeBuilder.nodeBuilder().settings(
ImmutableSettings.settingsBuilder().put(settings).put("node.master", true)
).node();
Node[] otherNodes = new Node[NUM_ADDITIONAL_NODES];
for (int i = 0; i < otherNodes.length; i++) {
otherNodes[i] = NodeBuilder.nodeBuilder().settings(settings).node();
}
Client client = node1.client();
try {
client.admin().indices().prepareCreate(INDEX_NAME).execute().actionGet();
} catch (IndexAlreadyExistsException e) {}
client.admin().cluster().prepareHealth().setWaitForYellowStatus().execute().actionGet();
int numberOfAliases = countAliases(client);
System.out.println("Number of aliases: " + numberOfAliases);
if (numberOfAliases < BASE_ALIAS_COUNT) {
int diff = BASE_ALIAS_COUNT - numberOfAliases;
System.out.println("Adding " + diff + " more aliases to get to the start amount of " + BASE_ALIAS_COUNT + " aliases");
IndicesAliasesRequestBuilder builder = client.admin().indices().prepareAliases();
for (int i = 1; i <= diff; i++) {
builder.addAlias(INDEX_NAME, "alias" + numberOfAliases + i);
if (i % 1000 == 0) {
builder.execute().actionGet();
builder = client.admin().indices().prepareAliases();
}
}
if (!builder.request().getAliasActions().isEmpty()) {
builder.execute().actionGet();
}
} else if (numberOfAliases > BASE_ALIAS_COUNT) {
IndicesAliasesRequestBuilder builder = client.admin().indices().prepareAliases();
int diff = numberOfAliases - BASE_ALIAS_COUNT;
System.out.println("Removing " + diff + " aliases to get to the start amount of " + BASE_ALIAS_COUNT + "aliases");
for (int i = 0; i <= diff; i++) {
builder.removeAlias(INDEX_NAME, "alias" + (BASE_ALIAS_COUNT + i));
if (i % 1000 == 0) {
builder.execute().actionGet();
builder = client.admin().indices().prepareAliases();
}
}
if (!builder.request().getAliasActions().isEmpty()) {
builder.execute().actionGet();
}
}
numberOfAliases = countAliases(client);
System.out.println("Number of aliases: " + numberOfAliases);
long totalTime = 0;
int max = numberOfAliases + NUM_ADD_ALIAS_REQUEST;
for (int i = numberOfAliases; i <= max; i++) {
if (i != numberOfAliases && i % 100 == 0) {
long avgTime = totalTime / 100;
System.out.println("Added [" + (i - numberOfAliases) + "] aliases. Avg create time: " + avgTime + " ms");
totalTime = 0;
}
long time = System.currentTimeMillis();
// String filter = termFilter("field" + i, "value" + i).toXContent(XContentFactory.jsonBuilder(), null).string();
client.admin().indices().prepareAliases().addAlias(INDEX_NAME, "alias" + i/*, filter*/)
.execute().actionGet();
totalTime += System.currentTimeMillis() - time;
}
System.out.println("Number of aliases: " + countAliases(client));
client.close();
node1.close();
for (Node otherNode : otherNodes) {
otherNode.close();
}
}
private static int countAliases(Client client) {
try {
GetAliasesResponse response = client.admin().indices().prepareGetAliases("a*")
.addIndices(INDEX_NAME)
.execute().actionGet();
return response.getAliases().get(INDEX_NAME).size();
} catch (AliasMissingException e) {
return 0;
}
}
}