diff --git a/solr/core/src/java/org/apache/solr/cloud/Overseer.java b/solr/core/src/java/org/apache/solr/cloud/Overseer.java index 2793db92a9f..9298216f301 100644 --- a/solr/core/src/java/org/apache/solr/cloud/Overseer.java +++ b/solr/core/src/java/org/apache/solr/cloud/Overseer.java @@ -46,6 +46,8 @@ import org.apache.zookeeper.KeeperException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static java.util.Collections.singletonMap; + /** * Cluster leader. Responsible node assignments, cluster state file? */ @@ -611,11 +613,11 @@ public class Overseer { List ranges = router.partitionRange(shards.size(), router.fullRange()); - Map newCollections = new LinkedHashMap(); +// Map newCollections = new LinkedHashMap(); Map newSlices = new LinkedHashMap(); - newCollections.putAll(state.getCollectionStates()); +// newCollections.putAll(state.getCollectionStates()); for (int i = 0; i < shards.size(); i++) { String sliceName = shards.get(i); /*} @@ -643,9 +645,10 @@ public class Overseer { if(message.getStr("fromApi") == null) collectionProps.put("autoCreated","true"); DocCollection newCollection = new DocCollection(collectionName, newSlices, collectionProps, router); - newCollections.put(collectionName, newCollection); - ClusterState newClusterState = new ClusterState(state.getLiveNodes(), newCollections); - return newClusterState; +// newCollections.put(collectionName, newCollection); + return state.copyWith(singletonMap(newCollection.getName(), newCollection)); +// ClusterState newClusterState = new ClusterState(state.getLiveNodes(), newCollections); +// return newClusterState; } /* @@ -771,6 +774,9 @@ public class Overseer { newCollections.put(collectionName, newCollection); return new ClusterState(state.getLiveNodes(), newCollections); } + private ClusterState newState(ClusterState state, Map colls) { + return state.copyWith(colls); + } /* * Remove collection from cloudstate @@ -779,11 +785,11 @@ public class Overseer { final String collection = message.getStr("name"); - final Map newCollections = new LinkedHashMap(clusterState.getCollectionStates()); // shallow copy - newCollections.remove(collection); +// final Map newCollections = new LinkedHashMap(clusterState.getCollectionStates()); // shallow copy +// newCollections.remove(collection); - ClusterState newState = new ClusterState(clusterState.getLiveNodes(), newCollections); - return newState; +// ClusterState newState = new ClusterState(clusterState.getLiveNodes(), newCollections); + return clusterState.copyWith(singletonMap(collection, (DocCollection)null)); } /* @@ -795,16 +801,17 @@ public class Overseer { log.info("Removing collection: " + collection + " shard: " + sliceId + " from clusterstate"); - final Map newCollections = new LinkedHashMap(clusterState.getCollectionStates()); // shallow copy - DocCollection coll = newCollections.get(collection); +// final Map newCollections = new LinkedHashMap(clusterState.getCollectionStates()); // shallow copy + DocCollection coll = clusterState.getCollection(collection); Map newSlices = new LinkedHashMap(coll.getSlicesMap()); newSlices.remove(sliceId); DocCollection newCollection = new DocCollection(coll.getName(), newSlices, coll.getProperties(), coll.getRouter()); - newCollections.put(newCollection.getName(), newCollection); +// newCollections.put(newCollection.getName(), newCollection); + return newState(clusterState, singletonMap(collection,newCollection)); - return new ClusterState(clusterState.getLiveNodes(), newCollections); +// return new ClusterState(clusterState.getLiveNodes(), newCollections); } /* @@ -816,8 +823,9 @@ public class Overseer { final String collection = message.getStr(ZkStateReader.COLLECTION_PROP); - final Map newCollections = new LinkedHashMap(clusterState.getCollectionStates()); // shallow copy - DocCollection coll = newCollections.get(collection); +// final Map newCollections = new LinkedHashMap(clusterState.getCollectionStates()); // shallow copy +// DocCollection coll = newCollections.get(collection); + DocCollection coll = clusterState.getCollectionOrNull(collection) ; if (coll == null) { // TODO: log/error that we didn't find it? // just in case, remove the zk collection node @@ -866,7 +874,7 @@ public class Overseer { // if there are no slices left in the collection, remove it? if (newSlices.size() == 0) { - newCollections.remove(coll.getName()); +// newCollections.remove(coll.getName()); // TODO: it might be better logically to have this in ZkController // but for tests (it's easier) it seems better for the moment to leave CoreContainer and/or @@ -879,15 +887,18 @@ public class Overseer { } catch (KeeperException e) { SolrException.log(log, "Problem cleaning up collection in zk:" + collection, e); } + return newState(clusterState,singletonMap(collection, (DocCollection) null)); + } else { DocCollection newCollection = new DocCollection(coll.getName(), newSlices, coll.getProperties(), coll.getRouter()); - newCollections.put(newCollection.getName(), newCollection); + return newState(clusterState,singletonMap(collection,newCollection)); +// newCollections.put(newCollection.getName(), newCollection); } - ClusterState newState = new ClusterState(clusterState.getLiveNodes(), newCollections); - return newState; +// ClusterState newState = new ClusterState(clusterState.getLiveNodes(), newCollections); +// return newState; } @Override diff --git a/solr/core/src/test/org/apache/solr/cloud/ClusterStateTest.java b/solr/core/src/test/org/apache/solr/cloud/ClusterStateTest.java index 092f647b879..0af40e11da4 100644 --- a/solr/core/src/test/org/apache/solr/cloud/ClusterStateTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/ClusterStateTest.java @@ -58,7 +58,7 @@ public class ClusterStateTest extends SolrTestCaseJ4 { ClusterState clusterState = new ClusterState(liveNodes, collectionStates); byte[] bytes = ZkStateReader.toJSON(clusterState); // System.out.println("#################### " + new String(bytes)); - ClusterState loadedClusterState = ClusterState.load(null, bytes, liveNodes); + ClusterState loadedClusterState = ClusterState.load(null, bytes, liveNodes,null); assertEquals("Provided liveNodes not used properly", 2, loadedClusterState .getLiveNodes().size()); @@ -66,13 +66,13 @@ public class ClusterStateTest extends SolrTestCaseJ4 { assertEquals("Poperties not copied properly", replica.getStr("prop1"), loadedClusterState.getSlice("collection1", "shard1").getReplicasMap().get("node1").getStr("prop1")); assertEquals("Poperties not copied properly", replica.getStr("prop2"), loadedClusterState.getSlice("collection1", "shard1").getReplicasMap().get("node1").getStr("prop2")); - loadedClusterState = ClusterState.load(null, new byte[0], liveNodes); + loadedClusterState = ClusterState.load(null, new byte[0], liveNodes,null); assertEquals("Provided liveNodes not used properly", 2, loadedClusterState .getLiveNodes().size()); assertEquals("Should not have collections", 0, loadedClusterState.getCollections().size()); - loadedClusterState = ClusterState.load(null, (byte[])null, liveNodes); + loadedClusterState = ClusterState.load(null, (byte[])null, liveNodes,null); assertEquals("Provided liveNodes not used properly", 2, loadedClusterState .getLiveNodes().size()); diff --git a/solr/core/src/test/org/apache/solr/cloud/SliceStateTest.java b/solr/core/src/test/org/apache/solr/cloud/SliceStateTest.java index 85b54daef60..2206f060981 100644 --- a/solr/core/src/test/org/apache/solr/cloud/SliceStateTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/SliceStateTest.java @@ -51,7 +51,7 @@ public class SliceStateTest extends SolrTestCaseJ4 { ClusterState clusterState = new ClusterState(liveNodes, collectionStates); byte[] bytes = ZkStateReader.toJSON(clusterState); - ClusterState loadedClusterState = ClusterState.load(null, bytes, liveNodes); + ClusterState loadedClusterState = ClusterState.load(null, bytes, liveNodes, null); assertEquals("Default state not set to active", "active", loadedClusterState.getSlice("collection1", "shard1").getState()); } diff --git a/solr/core/src/test/org/apache/solr/cloud/SliceStateUpdateTest.java b/solr/core/src/test/org/apache/solr/cloud/SliceStateUpdateTest.java index c72273efafe..8dde806418b 100644 --- a/solr/core/src/test/org/apache/solr/cloud/SliceStateUpdateTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/SliceStateUpdateTest.java @@ -37,6 +37,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; +import java.util.Collections; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.Map; @@ -142,8 +143,8 @@ public class SliceStateUpdateTest extends SolrTestCaseJ4 { closeThread(updaterThread); ClusterState clusterState = container1.getZkController().getClusterState(); - Map collectionStates = - new LinkedHashMap(clusterState.getCollectionStates()); +// Map collectionStates = +// new LinkedHashMap(clusterState.getCollectionStates()); Map slicesMap = clusterState.getSlicesMap("collection1"); Map props = new HashMap(1); @@ -155,11 +156,11 @@ public class SliceStateUpdateTest extends SolrTestCaseJ4 { props.put(DocCollection.DOC_ROUTER, ZkNodeProps.makeMap("name", ImplicitDocRouter.NAME)); DocCollection coll = new DocCollection("collection1", slicesMap, props, DocRouter.DEFAULT); - collectionStates.put("collection1", coll); +// collectionStates.put("collection1", coll); SolrZkClient zkClient = new SolrZkClient(zkServer.getZkAddress(), AbstractZkTestCase.TIMEOUT); - ClusterState newState = new ClusterState(clusterState.getLiveNodes(), collectionStates); + ClusterState newState = clusterState.copyWith(Collections.singletonMap(coll.getName(), coll) ); zkClient.setData(ZkStateReader.CLUSTER_STATE, ZkStateReader.toJSON(newState), true); zkClient.close(); diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterState.java b/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterState.java index a6719cc010f..29c6a87e8e7 100644 --- a/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterState.java +++ b/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterState.java @@ -45,7 +45,9 @@ public class ClusterState implements JSONWriter.Writable { private Integer zkClusterStateVersion; private final Map collectionStates; // Map> - private final Set liveNodes; + private Set liveNodes; + private final ZkStateReader stateReader; + /** * Use this constr when ClusterState is meant for publication. @@ -54,19 +56,43 @@ public class ClusterState implements JSONWriter.Writable { */ public ClusterState(Set liveNodes, Map collectionStates) { - this(null, liveNodes, collectionStates); + this(null, liveNodes, collectionStates, null); + } + + /** + * @deprecated + */ + public ClusterState(Integer zkClusterStateVersion, Set liveNodes, + Map collectionStates) { + this(zkClusterStateVersion, liveNodes, collectionStates,null); + } /** * Use this constr when ClusterState is meant for consumption. */ public ClusterState(Integer zkClusterStateVersion, Set liveNodes, - Map collectionStates) { + Map collectionStates, ZkStateReader stateReader) { this.zkClusterStateVersion = zkClusterStateVersion; this.liveNodes = new HashSet(liveNodes.size()); this.liveNodes.addAll(liveNodes); - this.collectionStates = new HashMap(collectionStates.size()); + this.collectionStates = new LinkedHashMap(collectionStates.size()); this.collectionStates.putAll(collectionStates); + this.stateReader = stateReader; + + } + + public ClusterState copyWith(Map modified){ + ClusterState result = new ClusterState(zkClusterStateVersion, liveNodes,collectionStates,stateReader); + for (Entry e : modified.entrySet()) { + DocCollection c = e.getValue(); + if(c == null) { + result.collectionStates.remove(e.getKey()); + continue; + } + result.collectionStates.put(c.getName(), c); + } + return result; } @@ -208,29 +234,28 @@ public class ClusterState implements JSONWriter.Writable { /** * Create ClusterState by reading the current state from zookeeper. */ - public static ClusterState load(SolrZkClient zkClient, Set liveNodes) throws KeeperException, InterruptedException { + public static ClusterState load(SolrZkClient zkClient, Set liveNodes, ZkStateReader stateReader) throws KeeperException, InterruptedException { Stat stat = new Stat(); byte[] state = zkClient.getData(ZkStateReader.CLUSTER_STATE, null, stat, true); - return load(stat.getVersion(), state, liveNodes); + return load(stat.getVersion(), state, liveNodes, stateReader); } /** * Create ClusterState from json string that is typically stored in zookeeper. * - * Use {@link ClusterState#load(SolrZkClient, Set)} instead, unless you want to + * Use {@link ClusterState#load(SolrZkClient, Set, ZkStateReader)} instead, unless you want to * do something more when getting the data - such as get the stat, set watch, etc. - * * @param version zk version of the clusterstate.json file (bytes) * @param bytes clusterstate.json as a byte array * @param liveNodes list of live nodes * @return the ClusterState */ - public static ClusterState load(Integer version, byte[] bytes, Set liveNodes) { + public static ClusterState load(Integer version, byte[] bytes, Set liveNodes, ZkStateReader stateReader) { // System.out.println("######## ClusterState.load:" + (bytes==null ? null : new String(bytes))); if (bytes == null || bytes.length == 0) { - return new ClusterState(version, liveNodes, Collections.emptyMap()); + return new ClusterState(version, liveNodes, Collections.emptyMap(),stateReader); } Map stateMap = (Map) ZkStateReader.fromJSON(bytes); Map collections = new LinkedHashMap(stateMap.size()); @@ -337,7 +362,11 @@ public class ClusterState implements JSONWriter.Writable { return true; } - - + /**Internal API used only by ZkStateReader + * @param liveNodes + */ + void setLiveNodes(Set liveNodes){ + this.liveNodes = liveNodes; + } } diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java index ee17f8ab270..e67a68776aa 100644 --- a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java +++ b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java @@ -254,7 +254,7 @@ public class ZkStateReader { byte[] data = zkClient.getData(CLUSTER_STATE, thisWatch, stat , true); Set ln = ZkStateReader.this.clusterState.getLiveNodes(); - ClusterState clusterState = ClusterState.load(stat.getVersion(), data, ln); + ClusterState clusterState = ClusterState.load(stat.getVersion(), data, ln,ZkStateReader.this); // update volatile ZkStateReader.this.clusterState = clusterState; } @@ -326,7 +326,7 @@ public class ZkStateReader { Set liveNodeSet = new HashSet(); liveNodeSet.addAll(liveNodes); - ClusterState clusterState = ClusterState.load(zkClient, liveNodeSet); + ClusterState clusterState = ClusterState.load(zkClient, liveNodeSet, ZkStateReader.this); this.clusterState = clusterState; zkClient.exists(ALIASES, @@ -393,12 +393,14 @@ public class ZkStateReader { if (!onlyLiveNodes) { log.info("Updating cloud state from ZooKeeper... "); - clusterState = ClusterState.load(zkClient, liveNodesSet); + clusterState = ClusterState.load(zkClient, liveNodesSet,this); } else { log.info("Updating live nodes from ZooKeeper... ({})", liveNodesSet.size()); - clusterState = new ClusterState( + clusterState = this.clusterState; + clusterState.setLiveNodes(liveNodesSet); + /*clusterState = new ClusterState( ZkStateReader.this.clusterState.getZkClusterStateVersion(), liveNodesSet, - ZkStateReader.this.clusterState.getCollectionStates()); + ZkStateReader.this.clusterState.getCollectionStates());*/ } this.clusterState = clusterState; } @@ -427,7 +429,7 @@ public class ZkStateReader { if (!onlyLiveNodes) { log.info("Updating cloud state from ZooKeeper... "); - clusterState = ClusterState.load(zkClient, liveNodesSet); + clusterState = ClusterState.load(zkClient, liveNodesSet,ZkStateReader.this); } else { log.info("Updating live nodes from ZooKeeper... "); clusterState = new ClusterState(ZkStateReader.this.clusterState.getZkClusterStateVersion(), liveNodesSet, ZkStateReader.this.clusterState.getCollectionStates());