From 12269abe349eff6655f2c3ba96bbad7667b1c641 Mon Sep 17 00:00:00 2001 From: Noble Paul Date: Fri, 1 Jun 2018 00:50:52 +1000 Subject: [PATCH] SOLR-12387: cluster-wide defaults for numShards, nrtReplicas, tlogReplicas, pullReplicas SOLR-12389: support deeply nested json objects in clusterprops.json --- solr/CHANGES.txt | 4 ++ .../handler/admin/CollectionHandlerApi.java | 58 +++++++++++++++---- .../handler/admin/CollectionsHandler.java | 11 ++++ .../solr/cloud/CollectionsAPISolrJTest.java | 51 ++++++++++++++++ .../handler/admin/TestCollectionAPIs.java | 5 ++ .../test/org/apache/solr/util/TestUtils.java | 28 ++++++++- .../solrj/request/CollectionAdminRequest.java | 2 +- .../solrj/request/CollectionApiMapping.java | 4 ++ .../solr/common/cloud/ClusterProperties.java | 18 +++++- .../solr/common/cloud/SolrZkClient.java | 33 +++++++++++ .../solr/common/cloud/ZkStateReader.java | 10 +++- .../solr/common/util/JsonSchemaValidator.java | 5 +- .../org/apache/solr/common/util/Utils.java | 42 ++++++++++++++ .../resources/apispec/cluster.Commands.json | 43 ++++++++++++++ .../solr/common/util/JsonValidatorTest.java | 8 +++ 15 files changed, 304 insertions(+), 18 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 9122b6d301b..af12aae095e 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -150,6 +150,10 @@ New Features collection. New /admin/metrics/history API allows retrieval of this data in numeric or graph formats. (ab) +* SOLR-12387: cluster-wide defaults for numShards, nrtReplicas, tlogReplicas, pullReplicas (noble) + +* SOLR-12389: support deeply nested json objects in clusterprops.json (noble) + Bug Fixes ---------------------- diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CollectionHandlerApi.java b/solr/core/src/java/org/apache/solr/handler/admin/CollectionHandlerApi.java index b2da1583587..d7d179ad56f 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/CollectionHandlerApi.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/CollectionHandlerApi.java @@ -17,6 +17,7 @@ package org.apache.solr.handler.admin; +import java.lang.invoke.MethodHandles; import java.util.Arrays; import java.util.Collection; import java.util.EnumMap; @@ -27,11 +28,19 @@ import org.apache.solr.client.solrj.request.CollectionApiMapping; import org.apache.solr.client.solrj.request.CollectionApiMapping.CommandMeta; import org.apache.solr.client.solrj.request.CollectionApiMapping.Meta; import org.apache.solr.client.solrj.request.CollectionApiMapping.V2EndPoint; +import org.apache.solr.common.Callable; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.cloud.ClusterProperties; +import org.apache.solr.common.util.CommandOperation; import org.apache.solr.handler.admin.CollectionsHandler.CollectionOperation; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class CollectionHandlerApi extends BaseHandlerApiSupport { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + final CollectionsHandler handler; static Collection apiCommands = createCollMapping(); @@ -55,26 +64,55 @@ public class CollectionHandlerApi extends BaseHandlerApiSupport { } } } - result.put(Meta.GET_NODES, new ApiCommand() { - @Override - public CommandMeta meta() { - return Meta.GET_NODES; - } + //The following APIs have only V2 implementations + addApi(result, Meta.GET_NODES, params -> params.rsp.add("nodes", ((CollectionHandlerApi) params.apiHandler).handler.coreContainer.getZkController().getClusterState().getLiveNodes())); + addApi(result, Meta.SET_CLUSTER_PROPERTY_OBJ, params -> { + List commands = params.req.getCommands(true); + if (commands == null || commands.isEmpty()) throw new RuntimeException("Empty commands"); + ClusterProperties clusterProperties = new ClusterProperties(((CollectionHandlerApi) params.apiHandler).handler.coreContainer.getZkController().getZkClient()); - @Override - public void invoke(SolrQueryRequest req, SolrQueryResponse rsp, BaseHandlerApiSupport apiHandler) throws Exception { - rsp.add("nodes", ((CollectionHandlerApi) apiHandler).handler.coreContainer.getZkController().getClusterState().getLiveNodes()); + try { + clusterProperties.setClusterProperties(commands.get(0).getDataMap()); + } catch (Exception e) { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error in API", e); } }); + for (Meta meta : Meta.values()) { - if(result.get(meta) == null){ - throw new RuntimeException("No implementation for "+ meta.name()); + if (result.get(meta) == null) { + log.error("ERROR_INIT. No corresponding API implementation for : " + meta.commandName); } } return result.values(); } + private static void addApi(Map result, Meta metaInfo, Callable fun) { + result.put(metaInfo, new ApiCommand() { + @Override + public CommandMeta meta() { + return metaInfo; + } + + @Override + public void invoke(SolrQueryRequest req, SolrQueryResponse rsp, BaseHandlerApiSupport apiHandler) throws Exception { + fun.call(new ApiParams(req, rsp, apiHandler)); + } + }); + } + + static class ApiParams { + final SolrQueryRequest req; + final SolrQueryResponse rsp; + final BaseHandlerApiSupport apiHandler; + + ApiParams(SolrQueryRequest req, SolrQueryResponse rsp, BaseHandlerApiSupport apiHandler) { + this.req = req; + this.rsp = rsp; + this.apiHandler = apiHandler; + } + } + public CollectionHandlerApi(CollectionsHandler handler) { this.handler = handler; } diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java index f7f6172ef8a..01d2fe89884 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java @@ -33,6 +33,7 @@ import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; @@ -122,6 +123,7 @@ import static org.apache.solr.common.cloud.DocCollection.RULE; import static org.apache.solr.common.cloud.DocCollection.SNITCH; import static org.apache.solr.common.cloud.DocCollection.STATE_FORMAT; import static org.apache.solr.common.cloud.ZkStateReader.AUTO_ADD_REPLICAS; +import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_DEF; import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP; import static org.apache.solr.common.cloud.ZkStateReader.MAX_SHARDS_PER_NODE; import static org.apache.solr.common.cloud.ZkStateReader.NRT_REPLICAS; @@ -204,6 +206,12 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission return this.coreContainer; } + protected void copyFromClusterProp(Map props, String prop) { + if (props.get(prop) != null) return;//if it's already specified , return + Object defVal = coreContainer.getZkController().getZkStateReader().getClusterProperty(ImmutableList.of(COLLECTION_DEF, prop), null); + if (defVal != null) props.put(prop, String.valueOf(defVal)); + } + @Override public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception { // Make sure the cores is enabled @@ -490,6 +498,9 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission createSysConfigSet(h.coreContainer); } + if (shardsParam == null) h.copyFromClusterProp(props, NUM_SLICES); + for (String prop : ImmutableSet.of(NRT_REPLICAS, PULL_REPLICAS, TLOG_REPLICAS)) + h.copyFromClusterProp(props, prop); copyPropertiesWithPrefix(req.getParams(), props, COLL_PROP_PREFIX); return copyPropertiesWithPrefix(req.getParams(), props, "router."); diff --git a/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java b/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java index 1a13f6c1785..d9e153ed055 100644 --- a/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java @@ -27,15 +27,19 @@ import java.util.Objects; import java.util.Optional; import java.util.concurrent.TimeUnit; +import com.google.common.collect.ImmutableList; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.TestUtil; +import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.impl.HttpSolrClient; import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.client.solrj.request.CoreAdminRequest; import org.apache.solr.client.solrj.request.CoreStatus; +import org.apache.solr.client.solrj.request.V2Request; import org.apache.solr.client.solrj.response.CollectionAdminResponse; import org.apache.solr.client.solrj.response.CoreAdminResponse; +import org.apache.solr.client.solrj.response.V2Response; import org.apache.solr.common.cloud.ClusterProperties; import org.apache.solr.common.cloud.DocCollection; import org.apache.solr.common.cloud.Replica; @@ -50,6 +54,10 @@ import org.apache.zookeeper.KeeperException; import org.junit.BeforeClass; import org.junit.Test; +import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_DEF; +import static org.apache.solr.common.cloud.ZkStateReader.NRT_REPLICAS; +import static org.apache.solr.common.cloud.ZkStateReader.NUM_SHARDS_PROP; + @LuceneTestCase.Slow public class CollectionsAPISolrJTest extends SolrCloudTestCase { @@ -91,6 +99,49 @@ public class CollectionsAPISolrJTest extends SolrCloudTestCase { waitForState("Expected " + collectionName + " to disappear from cluster state", collectionName, (n, c) -> c == null); } + @Test + public void testCreateCollWithDefaultClusterProperties() throws Exception { + String COLL_NAME = "CollWithDefaultClusterProperties"; + try { + V2Response rsp = new V2Request.Builder("/cluster") + .withMethod(SolrRequest.METHOD.POST) + .withPayload("{set-obj-property:{collectionDefaults:{numShards : 2 , nrtReplicas : 2}}}") + .build() + .process(cluster.getSolrClient()); + + for (int i = 0; i < 10; i++) { + Map m = cluster.getSolrClient().getZkStateReader().getClusterProperty(COLLECTION_DEF, null); + if (m != null) break; + Thread.sleep(10); + } + Object clusterProperty = cluster.getSolrClient().getZkStateReader().getClusterProperty(ImmutableList.of(COLLECTION_DEF, NUM_SHARDS_PROP), null); + assertEquals("2", String.valueOf(clusterProperty)); + clusterProperty = cluster.getSolrClient().getZkStateReader().getClusterProperty(ImmutableList.of(COLLECTION_DEF, NRT_REPLICAS), null); + assertEquals("2", String.valueOf(clusterProperty)); + CollectionAdminResponse response = CollectionAdminRequest + .createCollection(COLL_NAME, "conf", null, null, null, null) + .process(cluster.getSolrClient()); + assertEquals(0, response.getStatus()); + assertTrue(response.isSuccess()); + + DocCollection coll = cluster.getSolrClient().getClusterStateProvider().getClusterState().getCollection(COLL_NAME); + Map slices = coll.getSlicesMap(); + assertEquals(2, slices.size()); + for (Slice slice : slices.values()) { + assertEquals(2, slice.getReplicas().size()); + } + CollectionAdminRequest.deleteCollection(COLL_NAME).process(cluster.getSolrClient()); + } finally { + V2Response rsp = new V2Request.Builder("/cluster") + .withMethod(SolrRequest.METHOD.POST) + .withPayload("{set-obj-property:{collectionDefaults: null}}") + .build() + .process(cluster.getSolrClient()); + + } + + } + @Test public void testCreateAndDeleteCollection() throws Exception { String collectionName = "solrj_test"; diff --git a/solr/core/src/test/org/apache/solr/handler/admin/TestCollectionAPIs.java b/solr/core/src/test/org/apache/solr/handler/admin/TestCollectionAPIs.java index 3601347608d..0ac81a86b8c 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/TestCollectionAPIs.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/TestCollectionAPIs.java @@ -278,6 +278,11 @@ public class TestCollectionAPIs extends SolrTestCaseJ4 { MockCollectionsHandler() { } + @Override + protected void copyFromClusterProp(Map props, String prop) { + + } + @Override void invokeAction(SolrQueryRequest req, SolrQueryResponse rsp, CoreContainer cores, diff --git a/solr/core/src/test/org/apache/solr/util/TestUtils.java b/solr/core/src/test/org/apache/solr/util/TestUtils.java index 2f4cd53f963..edf813581c0 100644 --- a/solr/core/src/test/org/apache/solr/util/TestUtils.java +++ b/solr/core/src/test/org/apache/solr/util/TestUtils.java @@ -26,6 +26,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import com.google.common.collect.ImmutableList; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.common.MapWriter; import org.apache.solr.common.util.CommandOperation; @@ -39,6 +40,9 @@ import org.apache.solr.common.util.Utils; import org.junit.Assert; import static java.nio.charset.StandardCharsets.UTF_8; +import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_DEF; +import static org.apache.solr.common.cloud.ZkStateReader.NRT_REPLICAS; +import static org.apache.solr.common.cloud.ZkStateReader.NUM_SHARDS_PROP; import static org.apache.solr.common.util.Utils.fromJSONString; /** @@ -228,7 +232,7 @@ public class TestUtils extends SolrTestCaseJ4 { } public void testUtilsJSPath(){ - + String json = "{\n" + " 'authorization':{\n" + " 'class':'solr.RuleBasedAuthorizationPlugin',\n" + @@ -246,6 +250,26 @@ public class TestUtils extends SolrTestCaseJ4 { " '':{'v':4}}}"; Map m = (Map) fromJSONString(json); assertEquals("x-update", Utils.getObjectByPath(m,false, "authorization/permissions[1]/name")); - + + } + + public void testMergeJson() { + Map sink = (Map) Utils.fromJSONString("{k2:v2, k1: {a:b, p:r, k21:{xx:yy}}}"); + assertTrue(Utils.mergeJson(sink, (Map) Utils.fromJSONString("k1:{a:c, e:f, p :null, k11:{a1:b1}, k21:{pp : qq}}"))); + + assertEquals("v2", Utils.getObjectByPath(sink, true, "k2")); + assertEquals("c", Utils.getObjectByPath(sink, true, "k1/a")); + assertEquals("yy", Utils.getObjectByPath(sink, true, "k1/k21/xx")); + assertEquals("qq", Utils.getObjectByPath(sink, true, "k1/k21/pp")); + assertEquals("f", Utils.getObjectByPath(sink, true, "k1/e")); + assertEquals("b1", Utils.getObjectByPath(sink, true, "k1/k11/a1")); + + sink = new HashMap<>(); + sink.put("legacyCloud", "false"); + assertTrue(Utils.mergeJson(sink, (Map) Utils.fromJSONString("collectionDefaults:{numShards:3 , nrtReplicas:2}"))); + assertEquals(3l, Utils.getObjectByPath(sink, true, ImmutableList.of(COLLECTION_DEF, NUM_SHARDS_PROP))); + assertEquals(2l, Utils.getObjectByPath(sink, true, ImmutableList.of(COLLECTION_DEF, NRT_REPLICAS))); + + } } diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java index 4c74a58be27..4566033cfef 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java @@ -320,7 +320,7 @@ public abstract class CollectionAdminRequest * @param numTlogReplicas the number of {@link org.apache.solr.common.cloud.Replica.Type#TLOG} replicas * @param numPullReplicas the number of {@link org.apache.solr.common.cloud.Replica.Type#PULL} replicas */ - public static Create createCollection(String collection, String config, int numShards, int numNrtReplicas, int numTlogReplicas, int numPullReplicas) { + public static Create createCollection(String collection, String config, Integer numShards, Integer numNrtReplicas, Integer numTlogReplicas, Integer numPullReplicas) { return new Create(collection, config, numShards, numNrtReplicas, numTlogReplicas, numPullReplicas); } diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionApiMapping.java b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionApiMapping.java index 7fd4efe353f..74d0bbc175c 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionApiMapping.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionApiMapping.java @@ -203,6 +203,10 @@ public class CollectionApiMapping { POST, CLUSTERPROP, "set-property",null), + SET_CLUSTER_PROPERTY_OBJ(CLUSTER_CMD, + POST, + null, + "set-obj-property", null), UTILIZE_NODE(CLUSTER_CMD, POST, UTILIZENODE, diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterProperties.java b/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterProperties.java index e6df8459e35..87896daad5b 100644 --- a/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterProperties.java +++ b/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterProperties.java @@ -18,6 +18,7 @@ package org.apache.solr.common.cloud; import java.io.IOException; +import java.lang.invoke.MethodHandles; import java.util.Collections; import java.util.LinkedHashMap; import java.util.Map; @@ -27,6 +28,8 @@ import org.apache.solr.common.util.Utils; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.data.Stat; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Interact with solr cluster properties @@ -36,6 +39,8 @@ import org.apache.zookeeper.data.Stat; * {@link ZkStateReader#getClusterProperty(String, Object)} */ public class ClusterProperties { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + private final SolrZkClient client; @@ -48,7 +53,7 @@ public class ClusterProperties { /** * Read the value of a cluster property, returning a default if it is not set - * @param key the property name + * @param key the property name or the full path to the property. * @param defaultValue the default value * @param the type of the property * @return the property value @@ -56,7 +61,7 @@ public class ClusterProperties { */ @SuppressWarnings("unchecked") public T getClusterProperty(String key, T defaultValue) throws IOException { - T value = (T) getClusterProperties().get(key); + T value = (T) Utils.getObjectByPath(getClusterProperties(), false, key); if (value == null) return defaultValue; return value; @@ -77,6 +82,15 @@ public class ClusterProperties { } } + public void setClusterProperties(Map properties) throws IOException, KeeperException, InterruptedException { + client.atomicUpdate(ZkStateReader.CLUSTER_PROPS, zkData -> { + if (zkData == null) return Utils.toJSON(properties); + Map zkJson = (Map) Utils.fromJSON(zkData); + boolean modified = Utils.mergeJson(zkJson, properties); + return modified ? Utils.toJSON(zkJson) : null; + }); + } + /** * This method sets a cluster property. * diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZkClient.java b/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZkClient.java index c646258f00e..18750739b1c 100644 --- a/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZkClient.java +++ b/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZkClient.java @@ -34,6 +34,7 @@ import java.nio.file.Path; import java.util.List; import java.util.concurrent.ExecutorService; import java.util.concurrent.RejectedExecutionException; +import java.util.function.Function; import java.util.function.Predicate; import java.util.regex.Pattern; @@ -355,6 +356,38 @@ public class SolrZkClient implements Closeable { } } + public void atomicUpdate(String path, Function editor) throws KeeperException, InterruptedException { + for (; ; ) { + byte[] modified = null; + byte[] zkData = null; + Stat s = new Stat(); + try { + if (exists(path, true)) { + zkData = getData(path, null, s, true); + modified = editor.apply(zkData); + if (modified == null) { + //no change , no need to persist + return; + } + setData(path, modified, s.getVersion(), true); + break; + } else { + modified = editor.apply(null); + if (modified == null) { + //no change , no need to persist + return; + } + create(path, modified, CreateMode.PERSISTENT, true); + break; + } + } catch (KeeperException.BadVersionException | KeeperException.NodeExistsException e) { + continue; + } + } + + + } + /** * Returns path of created node */ diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java index fa526782be6..6b65c344d4c 100644 --- a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java +++ b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java @@ -127,6 +127,7 @@ public class ZkStateReader implements Closeable { public final static String CONFIGNAME_PROP="configName"; public static final String LEGACY_CLOUD = "legacyCloud"; + public static final String COLLECTION_DEF = "collectionDefaults"; public static final String URL_SCHEME = "urlScheme"; @@ -954,7 +955,14 @@ public class ZkStateReader implements Closeable { */ @SuppressWarnings("unchecked") public T getClusterProperty(String key, T defaultValue) { - T value = (T) clusterProperties.get(key); + T value = (T) Utils.getObjectByPath( clusterProperties, false, key); + if (value == null) + return defaultValue; + return value; + } + + public T getClusterProperty(List keyPath, T defaultValue) { + T value = (T) Utils.getObjectByPath( clusterProperties, false, keyPath); if (value == null) return defaultValue; return value; diff --git a/solr/solrj/src/java/org/apache/solr/common/util/JsonSchemaValidator.java b/solr/solrj/src/java/org/apache/solr/common/util/JsonSchemaValidator.java index 2a8d2d179d7..b1fcc914342 100644 --- a/solr/solrj/src/java/org/apache/solr/common/util/JsonSchemaValidator.java +++ b/solr/solrj/src/java/org/apache/solr/common/util/JsonSchemaValidator.java @@ -74,9 +74,10 @@ public class JsonSchemaValidator { return errs.isEmpty() ? null : errs; } - boolean validate(Object data, List errs){ + boolean validate(Object data, List errs) { + if (data == null) return true; for (Validator validator : validators) { - if(!validator.validate(data, errs)) { + if (!validator.validate(data, errs)) { return false; } } diff --git a/solr/solrj/src/java/org/apache/solr/common/util/Utils.java b/solr/solrj/src/java/org/apache/solr/common/util/Utils.java index 995d38ab46d..70c9d81b648 100644 --- a/solr/solrj/src/java/org/apache/solr/common/util/Utils.java +++ b/solr/solrj/src/java/org/apache/solr/common/util/Utils.java @@ -456,6 +456,48 @@ public class Utils { } } + /**Applies one json over other. The 'input' is applied over the sink + * The values in input isapplied over the values in 'sink' . If a value is 'null' + * that value is removed from sink + * + * @param sink the original json object to start with. Ensure that this Map is mutable + * @param input the json with new values + * @return whether there was any change made to sink or not. + */ + + public static boolean mergeJson(Map sink, Map input) { + boolean isModified = false; + for (Map.Entry e : input.entrySet()) { + if (sink.get(e.getKey()) != null) { + Object sinkVal = sink.get(e.getKey()); + if (e.getValue() == null) { + sink.remove(e.getKey()); + isModified = true; + } else { + if (e.getValue() instanceof Map) { + Map mapInputVal = (Map) e.getValue(); + if (sinkVal instanceof Map) { + if (mergeJson((Map) sinkVal, mapInputVal)) isModified = true; + } else { + sink.put(e.getKey(), mapInputVal); + isModified = true; + } + } else { + sink.put(e.getKey(), e.getValue()); + isModified = true; + } + + } + } else if (e.getValue() != null) { + sink.put(e.getKey(), e.getValue()); + isModified = true; + } + + } + + return isModified; + } + public static String getBaseUrlForNodeName(final String nodeName, String urlScheme) { final int _offset = nodeName.indexOf("_"); if (_offset < 0) { diff --git a/solr/solrj/src/resources/apispec/cluster.Commands.json b/solr/solrj/src/resources/apispec/cluster.Commands.json index d8718189065..7adc671cc8a 100644 --- a/solr/solrj/src/resources/apispec/cluster.Commands.json +++ b/solr/solrj/src/resources/apispec/cluster.Commands.json @@ -70,6 +70,49 @@ "val" ] }, + "set-obj-property": { + "type": "object", + "documentation": "https://lucene.apache.org/solr/guide/collections-api.html#clusterprop", + "description": "Add, edit, or delete a cluster-wide property.", + "properties": { + "legacyCloud": { + "type": "boolean" + }, + "urlScheme": { + "type": "string" + }, + "autoAddReplicas": { + "type": "boolean" + }, + "maxCoresPerNode": { + "type": "boolean" + }, + "location": { + "type": "string" + }, + "collectionDefaults": { + "type": "object", + "properties": { + "numShards": { + "type": "integer", + "description": "Default no:of shards for a collection" + }, + "tlogReplicas": { + "type": "integer", + "description": "Default no:of TLOG replicas" + }, + "pullReplicas": { + "type": "integer", + "description": "Default no:of PULL replicas" + }, + "nrtReplicas": { + "type": "integer", + "description": "Default no:of NRT replicas" + } + } + } + } + }, "utilize-node": { "type": "object", "documentation": "https://lucene.apache.org/solr/guide/collections-api.html#utilizenode", diff --git a/solr/solrj/src/test/org/apache/solr/common/util/JsonValidatorTest.java b/solr/solrj/src/test/org/apache/solr/common/util/JsonValidatorTest.java index fa6d080288c..b39c497ce91 100644 --- a/solr/solrj/src/test/org/apache/solr/common/util/JsonValidatorTest.java +++ b/solr/solrj/src/test/org/apache/solr/common/util/JsonValidatorTest.java @@ -40,6 +40,7 @@ public class JsonValidatorTest extends SolrTestCaseJ4 { checkSchema("core.config.Commands"); checkSchema("core.SchemaEdit"); checkSchema("cluster.configs.Commands"); + checkSchema("cluster.Commands"); } @@ -176,6 +177,13 @@ public class JsonValidatorTest extends SolrTestCaseJ4 { } + public void testNullObjectValue() { + ValidatingJsonMap spec = Utils.getSpec("cluster.Commands").getSpec(); + JsonSchemaValidator validator = new JsonSchemaValidator((Map) Utils.getObjectByPath(spec, false, "/commands/set-obj-property")); + List object = validator.validateJson(Utils.fromJSONString("{collectionDefaults: null}")); + assertNull(object); + } + private void checkSchema(String name) { ValidatingJsonMap spec = Utils.getSpec(name).getSpec(); Map commands = (Map) spec.get("commands");