From df7e3f55a4bbdf0c892af96794cf08b801452a1a Mon Sep 17 00:00:00 2001 From: kimchy Date: Wed, 30 Jun 2010 16:06:31 +0300 Subject: [PATCH] improve cluster blocks data structures for simpler usage --- config/elasticsearch.yml | 2 + .../cluster/state/ClusterStateRequest.java | 13 ++ .../state/TransportClusterStateAction.java | 4 +- .../cluster/block/ClusterBlock.java | 41 +++- .../cluster/block/ClusterBlockLevel.java | 2 + .../cluster/block/ClusterBlocks.java | 182 ++++++++---------- .../elasticsearch/gateway/GatewayService.java | 3 +- .../cluster/state/RestClusterStateAction.java | 32 +-- 8 files changed, 149 insertions(+), 130 deletions(-) diff --git a/config/elasticsearch.yml b/config/elasticsearch.yml index e69de29bb2d..eb1dfc111dc 100644 --- a/config/elasticsearch.yml +++ b/config/elasticsearch.yml @@ -0,0 +1,2 @@ +gateway: + recover_after_nodes: 3 diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateRequest.java b/modules/elasticsearch/src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateRequest.java index 54da446048b..87295b4dcd3 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateRequest.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateRequest.java @@ -38,6 +38,8 @@ public class ClusterStateRequest extends MasterNodeOperationRequest { private boolean filterMetaData = false; + private boolean filterBlocks = false; + private String[] filteredIndices = Strings.EMPTY_ARRAY; public ClusterStateRequest() { @@ -74,6 +76,15 @@ public class ClusterStateRequest extends MasterNodeOperationRequest { return this; } + public boolean filterBlocks() { + return filterBlocks; + } + + public ClusterStateRequest filterBlocks(boolean filterBlocks) { + this.filterBlocks = filterBlocks; + return this; + } + public String[] filteredIndices() { return filteredIndices; } @@ -87,6 +98,7 @@ public class ClusterStateRequest extends MasterNodeOperationRequest { filterRoutingTable = in.readBoolean(); filterNodes = in.readBoolean(); filterMetaData = in.readBoolean(); + filterBlocks = in.readBoolean(); int size = in.readVInt(); if (size > 0) { filteredIndices = new String[size]; @@ -100,6 +112,7 @@ public class ClusterStateRequest extends MasterNodeOperationRequest { out.writeBoolean(filterRoutingTable); out.writeBoolean(filterNodes); out.writeBoolean(filterMetaData); + out.writeBoolean(filterBlocks); out.writeVInt(filteredIndices.length); for (String filteredIndex : filteredIndices) { out.writeUTF(filteredIndex); diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java b/modules/elasticsearch/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java index d2199454099..b4b9de00717 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java @@ -71,6 +71,9 @@ public class TransportClusterStateAction extends TransportMasterNodeOperationAct if (!request.filterRoutingTable()) { builder.routingTable(currentState.routingTable()); } + if (!request.filterBlocks()) { + builder.blocks(currentState.blocks()); + } if (!request.filterMetaData()) { if (request.filteredIndices().length > 0) { MetaData.Builder mdBuilder = newMetaDataBuilder(); @@ -87,7 +90,6 @@ public class TransportClusterStateAction extends TransportMasterNodeOperationAct builder.metaData(currentState.metaData()); } } - builder.blocks(currentState.blocks()); return new ClusterStateResponse(clusterName, builder.build()); } } \ No newline at end of file diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/block/ClusterBlock.java b/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/block/ClusterBlock.java index da00bc126b1..46faeebcabd 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/block/ClusterBlock.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/block/ClusterBlock.java @@ -22,6 +22,8 @@ package org.elasticsearch.cluster.block; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Streamable; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.builder.XContentBuilder; import java.io.IOException; import java.io.Serializable; @@ -29,18 +31,21 @@ import java.io.Serializable; /** * @author kimchy (shay.banon) */ -public class ClusterBlock implements Serializable, Streamable { +public class ClusterBlock implements Serializable, Streamable, ToXContent { private int id; private String description; + private ClusterBlockLevel[] levels; + private ClusterBlock() { } - public ClusterBlock(int id, String description) { + public ClusterBlock(int id, String description, ClusterBlockLevel... levels) { this.id = id; this.description = description; + this.levels = levels; } public int id() { @@ -51,6 +56,30 @@ public class ClusterBlock implements Serializable, Streamable { return this.description; } + public ClusterBlockLevel[] levels() { + return this.levels(); + } + + public boolean contains(ClusterBlockLevel level) { + for (ClusterBlockLevel testLevel : levels) { + if (testLevel == level) { + return true; + } + } + return false; + } + + @Override public void toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(Integer.toString(id)); + builder.field("description", description); + builder.startArray("levels"); + for (ClusterBlockLevel level : levels) { + builder.value(level.name().toLowerCase()); + } + builder.endArray(); + builder.endObject(); + } + public static ClusterBlock readClusterBlock(StreamInput in) throws IOException { ClusterBlock block = new ClusterBlock(); block.readFrom(in); @@ -60,11 +89,19 @@ public class ClusterBlock implements Serializable, Streamable { @Override public void readFrom(StreamInput in) throws IOException { id = in.readVInt(); description = in.readUTF(); + levels = new ClusterBlockLevel[in.readVInt()]; + for (int i = 0; i < levels.length; i++) { + levels[i] = ClusterBlockLevel.fromId(in.readVInt()); + } } @Override public void writeTo(StreamOutput out) throws IOException { out.writeVInt(id); out.writeUTF(description); + out.writeVInt(levels.length); + for (ClusterBlockLevel level : levels) { + out.writeVInt(level.id()); + } } @Override public boolean equals(Object o) { diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/block/ClusterBlockLevel.java b/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/block/ClusterBlockLevel.java index 654addf783d..d8d9381ce6a 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/block/ClusterBlockLevel.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/block/ClusterBlockLevel.java @@ -29,6 +29,8 @@ public enum ClusterBlockLevel { WRITE(1), METADATA(2); + public static ClusterBlockLevel[] ALL = new ClusterBlockLevel[]{READ, WRITE, METADATA}; + private final int id; ClusterBlockLevel(int id) { diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/block/ClusterBlocks.java b/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/block/ClusterBlocks.java index e30771ad988..f7fc68c5ca9 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/block/ClusterBlocks.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/block/ClusterBlocks.java @@ -37,50 +37,56 @@ import java.util.Set; */ public class ClusterBlocks { - public static final ClusterBlocks EMPTY_CLUSTER_BLOCK = new ClusterBlocks(new ImmutableLevelHolder[]{ImmutableLevelHolder.EMPTY, ImmutableLevelHolder.EMPTY, ImmutableLevelHolder.EMPTY}); + public static final ClusterBlocks EMPTY_CLUSTER_BLOCK = new ClusterBlocks(ImmutableSet.of(), ImmutableMap.>of()); + + private final ImmutableSet global; + + private final ImmutableMap> indicesBlocks; private final ImmutableLevelHolder[] levelHolders; - ClusterBlocks(ImmutableLevelHolder[] levelHolders) { - this.levelHolders = levelHolders; + ClusterBlocks(ImmutableSet global, ImmutableMap> indicesBlocks) { + this.global = global; + this.indicesBlocks = indicesBlocks; + + levelHolders = new ImmutableLevelHolder[ClusterBlockLevel.values().length]; + for (ClusterBlockLevel level : ClusterBlockLevel.values()) { + ImmutableSet.Builder globalBuilder = ImmutableSet.builder(); + for (ClusterBlock block : global) { + if (block.contains(level)) { + globalBuilder.add(block); + } + } + + + ImmutableMap.Builder> indicesBuilder = ImmutableMap.builder(); + for (Map.Entry> entry : indicesBlocks.entrySet()) { + ImmutableSet.Builder indexBuilder = ImmutableSet.builder(); + for (ClusterBlock block : entry.getValue()) { + if (block.contains(level)) { + indexBuilder.add(block); + } + } + + indicesBuilder.put(entry.getKey(), indexBuilder.build()); + } + + levelHolders[level.id()] = new ImmutableLevelHolder(globalBuilder.build(), indicesBuilder.build()); + } + } + + public ImmutableSet global() { + return global; + } + + public ImmutableMap> indices() { + return indicesBlocks; } public ImmutableSet global(ClusterBlockLevel level) { return levelHolders[level.id()].global(); } - public boolean globalBlocked() { - for (ClusterBlockLevel level : ClusterBlockLevel.values()) { - if (!levelHolders[level.id()].global().isEmpty()) { - return true; - } - } - return false; - } - - public ImmutableSet indicesBlocked() { - Set indices = Sets.newHashSet(); - for (ClusterBlockLevel level : ClusterBlockLevel.values()) { - for (String index : indices(level).keySet()) { - ImmutableSet indexBlocks = indices(level).get(index); - if (indexBlocks != null && !indexBlocks.isEmpty()) { - indices.add(index); - } - } - } - return ImmutableSet.copyOf(indices); - } - - public ImmutableMap> indexBlocks(String index) { - ImmutableMap.Builder> builder = ImmutableMap.builder(); - for (ClusterBlockLevel level : ClusterBlockLevel.values()) { - if (indices(level).containsKey(index) && !indices(level).get(index).isEmpty()) { - builder.put(level, indices(level).get(index)); - } - } - return builder.build(); - } - public ImmutableMap> indices(ClusterBlockLevel level) { return levelHolders[level.id()].indices(); } @@ -152,111 +158,81 @@ public class ClusterBlocks { public static class Builder { - private LevelHolder[] levelHolders; + private Set global = Sets.newHashSet(); + + private Map> indices = Maps.newHashMap(); public Builder() { - levelHolders = new LevelHolder[ClusterBlockLevel.values().length]; - for (int i = 0; i < levelHolders.length; i++) { - levelHolders[i] = new LevelHolder(); - } } public Builder blocks(ClusterBlocks blocks) { - for (ClusterBlockLevel level : ClusterBlockLevel.values()) { - levelHolders[level.id()].global().addAll(blocks.global(level)); - for (Map.Entry> entry : blocks.indices(level).entrySet()) { - if (!levelHolders[level.id()].indices().containsKey(entry.getKey())) { - levelHolders[level.id()].indices().put(entry.getKey(), Sets.newHashSet()); - } - levelHolders[level.id()].indices().get(entry.getKey()).addAll(entry.getValue()); + global.addAll(blocks.global()); + for (Map.Entry> entry : blocks.indices().entrySet()) { + if (!indices.containsKey(entry.getKey())) { + indices.put(entry.getKey(), Sets.newHashSet()); } + indices.get(entry.getKey()).addAll(entry.getValue()); } return this; } public Builder addGlobalBlock(ClusterBlock block) { - for (ClusterBlockLevel level : ClusterBlockLevel.values()) { - addGlobalBlock(level, block); - } - return this; - } - - public Builder addGlobalBlock(ClusterBlockLevel level, ClusterBlock block) { - levelHolders[level.id()].global().add(block); + global.add(block); return this; } public Builder removeGlobalBlock(ClusterBlock block) { - for (ClusterBlockLevel level : ClusterBlockLevel.values()) { - removeGlobalBlock(level, block); + global.remove(block); + return this; + } + + public Builder addIndexBlock(String index, ClusterBlock block) { + if (!indices.containsKey(index)) { + indices.put(index, Sets.newHashSet()); } + indices.get(index).add(block); return this; } - public Builder removeGlobalBlock(ClusterBlockLevel level, ClusterBlock block) { - levelHolders[level.id()].global().remove(block); - return this; - } - - public Builder addIndexBlock(String index, ClusterBlockLevel level, ClusterBlock block) { - if (!levelHolders[level.id()].indices().containsKey(index)) { - levelHolders[level.id()].indices().put(index, Sets.newHashSet()); - } - levelHolders[level.id()].indices().get(index).add(block); - return this; - } - - public Builder removeIndexBlock(String index, ClusterBlockLevel level, ClusterBlock block) { - Set indexBlocks = levelHolders[level.id()].indices().get(index); - if (indexBlocks == null) { + public Builder removeIndexBlock(String index, ClusterBlock block) { + if (!indices.containsKey(index)) { return this; } - indexBlocks.remove(block); + indices.get(index).remove(block); return this; } public ClusterBlocks build() { - ImmutableLevelHolder[] holders = new ImmutableLevelHolder[ClusterBlockLevel.values().length]; - for (ClusterBlockLevel level : ClusterBlockLevel.values()) { - ImmutableMap.Builder> indicesBuilder = ImmutableMap.builder(); - for (Map.Entry> entry : levelHolders[level.id()].indices().entrySet()) { - indicesBuilder.put(entry.getKey(), ImmutableSet.copyOf(entry.getValue())); - } - holders[level.id()] = new ImmutableLevelHolder(ImmutableSet.copyOf(levelHolders[level.id()].global()), indicesBuilder.build()); + ImmutableMap.Builder> indicesBuilder = ImmutableMap.builder(); + for (Map.Entry> entry : indices.entrySet()) { + indicesBuilder.put(entry.getKey(), ImmutableSet.copyOf(entry.getValue())); } - return new ClusterBlocks(holders); + return new ClusterBlocks(ImmutableSet.copyOf(global), indicesBuilder.build()); } public static ClusterBlocks readClusterBlocks(StreamInput in) throws IOException { - ImmutableLevelHolder[] holders = new ImmutableLevelHolder[ClusterBlockLevel.values().length]; - for (ClusterBlockLevel level : ClusterBlockLevel.values()) { - ImmutableSet global = readBlockSet(in); - ImmutableMap.Builder> indicesBuilder = ImmutableMap.builder(); - int size = in.readVInt(); - for (int j = 0; j < size; j++) { - indicesBuilder.put(in.readUTF().intern(), readBlockSet(in)); - } - holders[level.id()] = new ImmutableLevelHolder(global, indicesBuilder.build()); + ImmutableSet global = readBlockSet(in); + ImmutableMap.Builder> indicesBuilder = ImmutableMap.builder(); + int size = in.readVInt(); + for (int j = 0; j < size; j++) { + indicesBuilder.put(in.readUTF().intern(), readBlockSet(in)); } - return new ClusterBlocks(holders); + return new ClusterBlocks(global, indicesBuilder.build()); } - public static void writeClusterBlocks(ClusterBlocks clusterBlock, StreamOutput out) throws IOException { - for (ClusterBlockLevel level : ClusterBlockLevel.values()) { - writeBlockSet(clusterBlock.global(level), out); - out.writeVInt(clusterBlock.indices(level).size()); - for (Map.Entry> entry : clusterBlock.indices(level).entrySet()) { - out.writeUTF(entry.getKey()); - writeBlockSet(entry.getValue(), out); - } + public static void writeClusterBlocks(ClusterBlocks blocks, StreamOutput out) throws IOException { + writeBlockSet(blocks.global(), out); + out.writeVInt(blocks.indices().size()); + for (Map.Entry> entry : blocks.indices().entrySet()) { + out.writeUTF(entry.getKey()); + writeBlockSet(entry.getValue(), out); } } private static void writeBlockSet(ImmutableSet blocks, StreamOutput out) throws IOException { out.writeVInt(blocks.size()); for (ClusterBlock block : blocks) { - out.writeVInt(block.id()); - out.writeUTF(block.description()); + block.writeTo(out); } } @@ -264,7 +240,7 @@ public class ClusterBlocks { ImmutableSet.Builder builder = ImmutableSet.builder(); int size = in.readVInt(); for (int i = 0; i < size; i++) { - builder.add(new ClusterBlock(in.readVInt(), in.readUTF().intern())); + builder.add(ClusterBlock.readClusterBlock(in)); } return builder.build(); } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/gateway/GatewayService.java b/modules/elasticsearch/src/main/java/org/elasticsearch/gateway/GatewayService.java index 1339f2ec448..a0273040806 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/gateway/GatewayService.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/gateway/GatewayService.java @@ -22,6 +22,7 @@ package org.elasticsearch.gateway; import org.elasticsearch.ElasticSearchException; import org.elasticsearch.cluster.*; import org.elasticsearch.cluster.block.ClusterBlock; +import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.block.ClusterBlocks; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; @@ -50,7 +51,7 @@ import static org.elasticsearch.common.util.concurrent.Executors.*; */ public class GatewayService extends AbstractLifecycleComponent implements ClusterStateListener { - public final ClusterBlock NOT_RECOVERED_FROM_GATEWAY_BLOCK = new ClusterBlock(1, "not recovered from gateway"); + public final ClusterBlock NOT_RECOVERED_FROM_GATEWAY_BLOCK = new ClusterBlock(1, "not recovered from gateway", ClusterBlockLevel.ALL); private final Gateway gateway; diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/rest/action/admin/cluster/state/RestClusterStateAction.java b/modules/elasticsearch/src/main/java/org/elasticsearch/rest/action/admin/cluster/state/RestClusterStateAction.java index d5d60ae965b..9dcbc8d4473 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/rest/action/admin/cluster/state/RestClusterStateAction.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/rest/action/admin/cluster/state/RestClusterStateAction.java @@ -26,7 +26,6 @@ import org.elasticsearch.client.Client; import org.elasticsearch.client.Requests; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.block.ClusterBlock; -import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.routing.IndexRoutingTable; @@ -67,6 +66,7 @@ public class RestClusterStateAction extends BaseRestHandler { clusterStateRequest.filterNodes(request.paramAsBoolean("filter_nodes", clusterStateRequest.filterNodes())); clusterStateRequest.filterRoutingTable(request.paramAsBoolean("filter_routing_table", clusterStateRequest.filterRoutingTable())); clusterStateRequest.filterMetaData(request.paramAsBoolean("filter_metadata", clusterStateRequest.filterMetaData())); + clusterStateRequest.filterBlocks(request.paramAsBoolean("filter_blocks", clusterStateRequest.filterBlocks())); clusterStateRequest.filteredIndices(RestActions.splitIndices(request.param("filter_indices", null))); client.admin().cluster().state(new ClusterStateRequest(), new ActionListener() { @Override public void onResponse(ClusterStateResponse response) { @@ -82,34 +82,20 @@ public class RestClusterStateAction extends BaseRestHandler { // blocks builder.startObject("blocks"); - if (state.blocks().globalBlocked()) { + if (!state.blocks().global().isEmpty()) { builder.startObject("global"); - for (ClusterBlockLevel level : ClusterBlockLevel.values()) { - if (!state.blocks().global(level).isEmpty()) { - builder.startObject(level.name().toLowerCase()); - for (ClusterBlock block : state.blocks().global(level)) { - builder.startObject(Integer.toString(block.id())); - builder.field("description", block.description()); - builder.endObject(); - } - builder.endObject(); - } + for (ClusterBlock block : state.blocks().global()) { + block.toXContent(builder, request); } builder.endObject(); } - if (!state.blocks().indicesBlocked().isEmpty()) { + if (!state.blocks().indices().isEmpty()) { builder.startObject("indices"); - for (String index : state.blocks().indicesBlocked()) { - builder.startObject(index); - for (Map.Entry> entry : state.blocks().indexBlocks(index).entrySet()) { - ClusterBlockLevel level = entry.getKey(); - builder.startObject(level.name().toLowerCase()); - for (ClusterBlock block : entry.getValue()) { - builder.startObject(Integer.toString(block.id())); - builder.field("description", block.description()); - builder.endObject(); - } + for (Map.Entry> entry : state.blocks().indices().entrySet()) { + builder.startObject(entry.getKey()); + for (ClusterBlock block : entry.getValue()) { + block.toXContent(builder, request); } builder.endObject(); }