Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-12387: cluster-wide defaults for numShards, nrtReplicas, tlogReplicas, pullReplicas 

SOLR-12389: support deeply nested json objects in clusterprops.json
This commit is contained in:
Noble Paul 2018-06-01 00:50:52 +10:00
parent 76263087b5
commit 12269abe34
15 changed files with 304 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
solr/CHANGES.txt
View File

@ -150,6 +150,10 @@ New Features
collection. New /admin/metrics/history API allows retrieval of this data in numeric
or graph formats. (ab)
* SOLR-12387: cluster-wide defaults for numShards, nrtReplicas, tlogReplicas, pullReplicas (noble)
* SOLR-12389: support deeply nested json objects in clusterprops.json (noble)
Bug Fixes
----------------------

58
solr/core/src/java/org/apache/solr/handler/admin/CollectionHandlerApi.java
View File

@ -17,6 +17,7 @@
package org.apache.solr.handler.admin;
import java.lang.invoke.MethodHandles;
import java.util.Arrays;
import java.util.Collection;
import java.util.EnumMap;
@ -27,11 +28,19 @@ import org.apache.solr.client.solrj.request.CollectionApiMapping;
import org.apache.solr.client.solrj.request.CollectionApiMapping.CommandMeta;
import org.apache.solr.client.solrj.request.CollectionApiMapping.Meta;
import org.apache.solr.client.solrj.request.CollectionApiMapping.V2EndPoint;
import org.apache.solr.common.Callable;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.ClusterProperties;
import org.apache.solr.common.util.CommandOperation;
import org.apache.solr.handler.admin.CollectionsHandler.CollectionOperation;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class CollectionHandlerApi extends BaseHandlerApiSupport {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
final CollectionsHandler handler;
static Collection<ApiCommand> apiCommands = createCollMapping();
@ -55,26 +64,55 @@ public class CollectionHandlerApi extends BaseHandlerApiSupport {
}
}
}
result.put(Meta.GET_NODES, new ApiCommand() {
@Override
public CommandMeta meta() {
return Meta.GET_NODES;
}
//The following APIs have only V2 implementations
addApi(result, Meta.GET_NODES, params -> params.rsp.add("nodes", ((CollectionHandlerApi) params.apiHandler).handler.coreContainer.getZkController().getClusterState().getLiveNodes()));
addApi(result, Meta.SET_CLUSTER_PROPERTY_OBJ, params -> {
List<CommandOperation> commands = params.req.getCommands(true);
if (commands == null || commands.isEmpty()) throw new RuntimeException("Empty commands");
ClusterProperties clusterProperties = new ClusterProperties(((CollectionHandlerApi) params.apiHandler).handler.coreContainer.getZkController().getZkClient());
@Override
public void invoke(SolrQueryRequest req, SolrQueryResponse rsp, BaseHandlerApiSupport apiHandler) throws Exception {
rsp.add("nodes", ((CollectionHandlerApi) apiHandler).handler.coreContainer.getZkController().getClusterState().getLiveNodes());
try {
clusterProperties.setClusterProperties(commands.get(0).getDataMap());
} catch (Exception e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error in API", e);
}
});
for (Meta meta : Meta.values()) {
if(result.get(meta) == null){
throw new RuntimeException("No implementation for "+ meta.name());
if (result.get(meta) == null) {
log.error("ERROR_INIT. No corresponding API implementation for : " + meta.commandName);
}
}
return result.values();
}
private static void addApi(Map<Meta, ApiCommand> result, Meta metaInfo, Callable<ApiParams> fun) {
result.put(metaInfo, new ApiCommand() {
@Override
public CommandMeta meta() {
return metaInfo;
}
@Override
public void invoke(SolrQueryRequest req, SolrQueryResponse rsp, BaseHandlerApiSupport apiHandler) throws Exception {
fun.call(new ApiParams(req, rsp, apiHandler));
}
});
}
static class ApiParams {
final SolrQueryRequest req;
final SolrQueryResponse rsp;
final BaseHandlerApiSupport apiHandler;
ApiParams(SolrQueryRequest req, SolrQueryResponse rsp, BaseHandlerApiSupport apiHandler) {
this.req = req;
this.rsp = rsp;
this.apiHandler = apiHandler;
}
}
public CollectionHandlerApi(CollectionsHandler handler) {
this.handler = handler;
}

11
solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
View File

@ -33,6 +33,7 @@ import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
@ -122,6 +123,7 @@ import static org.apache.solr.common.cloud.DocCollection.RULE;
import static org.apache.solr.common.cloud.DocCollection.SNITCH;
import static org.apache.solr.common.cloud.DocCollection.STATE_FORMAT;
import static org.apache.solr.common.cloud.ZkStateReader.AUTO_ADD_REPLICAS;
import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_DEF;
import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP;
import static org.apache.solr.common.cloud.ZkStateReader.MAX_SHARDS_PER_NODE;
import static org.apache.solr.common.cloud.ZkStateReader.NRT_REPLICAS;
@ -204,6 +206,12 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission
return this.coreContainer;
}
protected void copyFromClusterProp(Map<String, Object> props, String prop) {
if (props.get(prop) != null) return;//if it's already specified , return
Object defVal = coreContainer.getZkController().getZkStateReader().getClusterProperty(ImmutableList.of(COLLECTION_DEF, prop), null);
if (defVal != null) props.put(prop, String.valueOf(defVal));
}
@Override
public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {
// Make sure the cores is enabled
@ -490,6 +498,9 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission
createSysConfigSet(h.coreContainer);
}
if (shardsParam == null) h.copyFromClusterProp(props, NUM_SLICES);
for (String prop : ImmutableSet.of(NRT_REPLICAS, PULL_REPLICAS, TLOG_REPLICAS))
h.copyFromClusterProp(props, prop);
copyPropertiesWithPrefix(req.getParams(), props, COLL_PROP_PREFIX);
return copyPropertiesWithPrefix(req.getParams(), props, "router.");

51
solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java
View File

@ -27,15 +27,19 @@ import java.util.Objects;
import java.util.Optional;
import java.util.concurrent.TimeUnit;
import com.google.common.collect.ImmutableList;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.client.solrj.request.CoreAdminRequest;
import org.apache.solr.client.solrj.request.CoreStatus;
import org.apache.solr.client.solrj.request.V2Request;
import org.apache.solr.client.solrj.response.CollectionAdminResponse;
import org.apache.solr.client.solrj.response.CoreAdminResponse;
import org.apache.solr.client.solrj.response.V2Response;
import org.apache.solr.common.cloud.ClusterProperties;
import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.Replica;
@ -50,6 +54,10 @@ import org.apache.zookeeper.KeeperException;
import org.junit.BeforeClass;
import org.junit.Test;
import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_DEF;
import static org.apache.solr.common.cloud.ZkStateReader.NRT_REPLICAS;
import static org.apache.solr.common.cloud.ZkStateReader.NUM_SHARDS_PROP;
@LuceneTestCase.Slow
public class CollectionsAPISolrJTest extends SolrCloudTestCase {
@ -91,6 +99,49 @@ public class CollectionsAPISolrJTest extends SolrCloudTestCase {
waitForState("Expected " + collectionName + " to disappear from cluster state", collectionName, (n, c) -> c == null);
}
@Test
public void testCreateCollWithDefaultClusterProperties() throws Exception {
String COLL_NAME = "CollWithDefaultClusterProperties";
try {
V2Response rsp = new V2Request.Builder("/cluster")
.withMethod(SolrRequest.METHOD.POST)
.withPayload("{set-obj-property:{collectionDefaults:{numShards : 2 , nrtReplicas : 2}}}")
.build()
.process(cluster.getSolrClient());
for (int i = 0; i < 10; i++) {
Map m = cluster.getSolrClient().getZkStateReader().getClusterProperty(COLLECTION_DEF, null);
if (m != null) break;
Thread.sleep(10);
}
Object clusterProperty = cluster.getSolrClient().getZkStateReader().getClusterProperty(ImmutableList.of(COLLECTION_DEF, NUM_SHARDS_PROP), null);
assertEquals("2", String.valueOf(clusterProperty));
clusterProperty = cluster.getSolrClient().getZkStateReader().getClusterProperty(ImmutableList.of(COLLECTION_DEF, NRT_REPLICAS), null);
assertEquals("2", String.valueOf(clusterProperty));
CollectionAdminResponse response = CollectionAdminRequest
.createCollection(COLL_NAME, "conf", null, null, null, null)
.process(cluster.getSolrClient());
assertEquals(0, response.getStatus());
assertTrue(response.isSuccess());
DocCollection coll = cluster.getSolrClient().getClusterStateProvider().getClusterState().getCollection(COLL_NAME);
Map<String, Slice> slices = coll.getSlicesMap();
assertEquals(2, slices.size());
for (Slice slice : slices.values()) {
assertEquals(2, slice.getReplicas().size());
}
CollectionAdminRequest.deleteCollection(COLL_NAME).process(cluster.getSolrClient());
} finally {
V2Response rsp = new V2Request.Builder("/cluster")
.withMethod(SolrRequest.METHOD.POST)
.withPayload("{set-obj-property:{collectionDefaults: null}}")
.build()
.process(cluster.getSolrClient());
}
}
@Test
public void testCreateAndDeleteCollection() throws Exception {
String collectionName = "solrj_test";

5
solr/core/src/test/org/apache/solr/handler/admin/TestCollectionAPIs.java
View File

@ -278,6 +278,11 @@ public class TestCollectionAPIs extends SolrTestCaseJ4 {
MockCollectionsHandler() {
}
@Override
protected void copyFromClusterProp(Map<String, Object> props, String prop) {
}
@Override
void invokeAction(SolrQueryRequest req, SolrQueryResponse rsp,
CoreContainer cores,

28
solr/core/src/test/org/apache/solr/util/TestUtils.java
View File

@ -26,6 +26,7 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import com.google.common.collect.ImmutableList;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.common.MapWriter;
import org.apache.solr.common.util.CommandOperation;
@ -39,6 +40,9 @@ import org.apache.solr.common.util.Utils;
import org.junit.Assert;
import static java.nio.charset.StandardCharsets.UTF_8;
import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_DEF;
import static org.apache.solr.common.cloud.ZkStateReader.NRT_REPLICAS;
import static org.apache.solr.common.cloud.ZkStateReader.NUM_SHARDS_PROP;
import static org.apache.solr.common.util.Utils.fromJSONString;
/**
@ -228,7 +232,7 @@ public class TestUtils extends SolrTestCaseJ4 {
}
public void testUtilsJSPath(){
String json = "{\n" +
" 'authorization':{\n" +
" 'class':'solr.RuleBasedAuthorizationPlugin',\n" +
@ -246,6 +250,26 @@ public class TestUtils extends SolrTestCaseJ4 {
" '':{'v':4}}}";
Map m = (Map) fromJSONString(json);
assertEquals("x-update", Utils.getObjectByPath(m,false, "authorization/permissions[1]/name"));
}
public void testMergeJson() {
Map<String, Object> sink = (Map<String, Object>) Utils.fromJSONString("{k2:v2, k1: {a:b, p:r, k21:{xx:yy}}}");
assertTrue(Utils.mergeJson(sink, (Map<String, Object>) Utils.fromJSONString("k1:{a:c, e:f, p :null, k11:{a1:b1}, k21:{pp : qq}}")));
assertEquals("v2", Utils.getObjectByPath(sink, true, "k2"));
assertEquals("c", Utils.getObjectByPath(sink, true, "k1/a"));
assertEquals("yy", Utils.getObjectByPath(sink, true, "k1/k21/xx"));
assertEquals("qq", Utils.getObjectByPath(sink, true, "k1/k21/pp"));
assertEquals("f", Utils.getObjectByPath(sink, true, "k1/e"));
assertEquals("b1", Utils.getObjectByPath(sink, true, "k1/k11/a1"));
sink = new HashMap<>();
sink.put("legacyCloud", "false");
assertTrue(Utils.mergeJson(sink, (Map<String, Object>) Utils.fromJSONString("collectionDefaults:{numShards:3 , nrtReplicas:2}")));
assertEquals(3l, Utils.getObjectByPath(sink, true, ImmutableList.of(COLLECTION_DEF, NUM_SHARDS_PROP)));
assertEquals(2l, Utils.getObjectByPath(sink, true, ImmutableList.of(COLLECTION_DEF, NRT_REPLICAS)));
}
}

2
solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
View File

@ -320,7 +320,7 @@ public abstract class CollectionAdminRequest<T extends CollectionAdminResponse>
* @param numTlogReplicas the number of {@link org.apache.solr.common.cloud.Replica.Type#TLOG} replicas
* @param numPullReplicas the number of {@link org.apache.solr.common.cloud.Replica.Type#PULL} replicas
*/
public static Create createCollection(String collection, String config, int numShards, int numNrtReplicas, int numTlogReplicas, int numPullReplicas) {
public static Create createCollection(String collection, String config, Integer numShards, Integer numNrtReplicas, Integer numTlogReplicas, Integer numPullReplicas) {
return new Create(collection, config, numShards, numNrtReplicas, numTlogReplicas, numPullReplicas);
}

4
solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionApiMapping.java
View File

@ -203,6 +203,10 @@ public class CollectionApiMapping {
POST,
CLUSTERPROP,
"set-property",null),
SET_CLUSTER_PROPERTY_OBJ(CLUSTER_CMD,
POST,
null,
"set-obj-property", null),
UTILIZE_NODE(CLUSTER_CMD,
POST,
UTILIZENODE,

18
solr/solrj/src/java/org/apache/solr/common/cloud/ClusterProperties.java
View File

@ -18,6 +18,7 @@
package org.apache.solr.common.cloud;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.Map;
@ -27,6 +28,8 @@ import org.apache.solr.common.util.Utils;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.data.Stat;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Interact with solr cluster properties
@ -36,6 +39,8 @@ import org.apache.zookeeper.data.Stat;
* {@link ZkStateReader#getClusterProperty(String, Object)}
*/
public class ClusterProperties {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private final SolrZkClient client;
@ -48,7 +53,7 @@ public class ClusterProperties {
/**
* Read the value of a cluster property, returning a default if it is not set
* @param key the property name
* @param key the property name or the full path to the property.
* @param defaultValue the default value
* @param <T> the type of the property
* @return the property value
@ -56,7 +61,7 @@ public class ClusterProperties {
*/
@SuppressWarnings("unchecked")
public <T> T getClusterProperty(String key, T defaultValue) throws IOException {
T value = (T) getClusterProperties().get(key);
T value = (T) Utils.getObjectByPath(getClusterProperties(), false, key);
if (value == null)
return defaultValue;
return value;
@ -77,6 +82,15 @@ public class ClusterProperties {
}
}
public void setClusterProperties(Map<String, Object> properties) throws IOException, KeeperException, InterruptedException {
client.atomicUpdate(ZkStateReader.CLUSTER_PROPS, zkData -> {
if (zkData == null) return Utils.toJSON(properties);
Map<String, Object> zkJson = (Map<String, Object>) Utils.fromJSON(zkData);
boolean modified = Utils.mergeJson(zkJson, properties);
return modified ? Utils.toJSON(zkJson) : null;
});
}
/**
* This method sets a cluster property.
*

33
solr/solrj/src/java/org/apache/solr/common/cloud/SolrZkClient.java
View File

@ -34,6 +34,7 @@ import java.nio.file.Path;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.RejectedExecutionException;
import java.util.function.Function;
import java.util.function.Predicate;
import java.util.regex.Pattern;
@ -355,6 +356,38 @@ public class SolrZkClient implements Closeable {
}
}
public void atomicUpdate(String path, Function<byte[], byte[]> editor) throws KeeperException, InterruptedException {
for (; ; ) {
byte[] modified = null;
byte[] zkData = null;
Stat s = new Stat();
try {
if (exists(path, true)) {
zkData = getData(path, null, s, true);
modified = editor.apply(zkData);
if (modified == null) {
//no change , no need to persist
return;
}
setData(path, modified, s.getVersion(), true);
break;
} else {
modified = editor.apply(null);
if (modified == null) {
//no change , no need to persist
return;
}
create(path, modified, CreateMode.PERSISTENT, true);
break;
}
} catch (KeeperException.BadVersionException | KeeperException.NodeExistsException e) {
continue;
}
}
}
/**
* Returns path of created node
*/

10
solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
View File

@ -127,6 +127,7 @@ public class ZkStateReader implements Closeable {
public final static String CONFIGNAME_PROP="configName";
public static final String LEGACY_CLOUD = "legacyCloud";
public static final String COLLECTION_DEF = "collectionDefaults";
public static final String URL_SCHEME = "urlScheme";
@ -954,7 +955,14 @@ public class ZkStateReader implements Closeable {
*/
@SuppressWarnings("unchecked")
public <T> T getClusterProperty(String key, T defaultValue) {
T value = (T) clusterProperties.get(key);
T value = (T) Utils.getObjectByPath( clusterProperties, false, key);
if (value == null)
return defaultValue;
return value;
}
public <T> T getClusterProperty(List<String> keyPath, T defaultValue) {
T value = (T) Utils.getObjectByPath( clusterProperties, false, keyPath);
if (value == null)
return defaultValue;
return value;

5
solr/solrj/src/java/org/apache/solr/common/util/JsonSchemaValidator.java
View File

@ -74,9 +74,10 @@ public class JsonSchemaValidator {
return errs.isEmpty() ? null : errs;
}
boolean validate(Object data, List<String> errs){
boolean validate(Object data, List<String> errs) {
if (data == null) return true;
for (Validator validator : validators) {
if(!validator.validate(data, errs)) {
if (!validator.validate(data, errs)) {
return false;
}
}

42
solr/solrj/src/java/org/apache/solr/common/util/Utils.java
View File

@ -456,6 +456,48 @@ public class Utils {
}
}
/**Applies one json over other. The 'input' is applied over the sink
* The values in input isapplied over the values in 'sink' . If a value is 'null'
* that value is removed from sink
*
* @param sink the original json object to start with. Ensure that this Map is mutable
* @param input the json with new values
* @return whether there was any change made to sink or not.
*/
public static boolean mergeJson(Map<String, Object> sink, Map<String, Object> input) {
boolean isModified = false;
for (Map.Entry<String, Object> e : input.entrySet()) {
if (sink.get(e.getKey()) != null) {
Object sinkVal = sink.get(e.getKey());
if (e.getValue() == null) {
sink.remove(e.getKey());
isModified = true;
} else {
if (e.getValue() instanceof Map) {
Map<String, Object> mapInputVal = (Map<String, Object>) e.getValue();
if (sinkVal instanceof Map) {
if (mergeJson((Map<String, Object>) sinkVal, mapInputVal)) isModified = true;
} else {
sink.put(e.getKey(), mapInputVal);
isModified = true;
}
} else {
sink.put(e.getKey(), e.getValue());
isModified = true;
}
}
} else if (e.getValue() != null) {
sink.put(e.getKey(), e.getValue());
isModified = true;
}
}
return isModified;
}
public static String getBaseUrlForNodeName(final String nodeName, String urlScheme) {
final int _offset = nodeName.indexOf("_");
if (_offset < 0) {

43
solr/solrj/src/resources/apispec/cluster.Commands.json
View File

@ -70,6 +70,49 @@
"val"
]
},
"set-obj-property": {
"type": "object",
"documentation": "https://lucene.apache.org/solr/guide/collections-api.html#clusterprop",
"description": "Add, edit, or delete a cluster-wide property.",
"properties": {
"legacyCloud": {
"type": "boolean"
},
"urlScheme": {
"type": "string"
},
"autoAddReplicas": {
"type": "boolean"
},
"maxCoresPerNode": {
"type": "boolean"
},
"location": {
"type": "string"
},
"collectionDefaults": {
"type": "object",
"properties": {
"numShards": {
"type": "integer",
"description": "Default no:of shards for a collection"
},
"tlogReplicas": {
"type": "integer",
"description": "Default no:of TLOG replicas"
},
"pullReplicas": {
"type": "integer",
"description": "Default no:of PULL replicas"
},
"nrtReplicas": {
"type": "integer",
"description": "Default no:of NRT replicas"
}
}
}
}
},
"utilize-node": {
"type": "object",
"documentation": "https://lucene.apache.org/solr/guide/collections-api.html#utilizenode",

8
solr/solrj/src/test/org/apache/solr/common/util/JsonValidatorTest.java
View File

@ -40,6 +40,7 @@ public class JsonValidatorTest extends SolrTestCaseJ4 {
checkSchema("core.config.Commands");
checkSchema("core.SchemaEdit");
checkSchema("cluster.configs.Commands");
checkSchema("cluster.Commands");
}
@ -176,6 +177,13 @@ public class JsonValidatorTest extends SolrTestCaseJ4 {
}
public void testNullObjectValue() {
ValidatingJsonMap spec = Utils.getSpec("cluster.Commands").getSpec();
JsonSchemaValidator validator = new JsonSchemaValidator((Map) Utils.getObjectByPath(spec, false, "/commands/set-obj-property"));
List<String> object = validator.validateJson(Utils.fromJSONString("{collectionDefaults: null}"));
assertNull(object);
}
private void checkSchema(String name) {
ValidatingJsonMap spec = Utils.getSpec(name).getSpec();
Map commands = (Map) spec.get("commands");