mirror of https://github.com/apache/lucene.git
SOLR-12387: cluster-wide defaults for numShards, nrtReplicas, tlogReplicas, pullReplicas
SOLR-12389: support deeply nested json objects in clusterprops.json
This commit is contained in:
parent
76263087b5
commit
12269abe34
|
@ -150,6 +150,10 @@ New Features
|
||||||
collection. New /admin/metrics/history API allows retrieval of this data in numeric
|
collection. New /admin/metrics/history API allows retrieval of this data in numeric
|
||||||
or graph formats. (ab)
|
or graph formats. (ab)
|
||||||
|
|
||||||
|
* SOLR-12387: cluster-wide defaults for numShards, nrtReplicas, tlogReplicas, pullReplicas (noble)
|
||||||
|
|
||||||
|
* SOLR-12389: support deeply nested json objects in clusterprops.json (noble)
|
||||||
|
|
||||||
Bug Fixes
|
Bug Fixes
|
||||||
----------------------
|
----------------------
|
||||||
|
|
||||||
|
|
|
@ -17,6 +17,7 @@
|
||||||
|
|
||||||
package org.apache.solr.handler.admin;
|
package org.apache.solr.handler.admin;
|
||||||
|
|
||||||
|
import java.lang.invoke.MethodHandles;
|
||||||
import java.util.Arrays;
|
import java.util.Arrays;
|
||||||
import java.util.Collection;
|
import java.util.Collection;
|
||||||
import java.util.EnumMap;
|
import java.util.EnumMap;
|
||||||
|
@ -27,11 +28,19 @@ import org.apache.solr.client.solrj.request.CollectionApiMapping;
|
||||||
import org.apache.solr.client.solrj.request.CollectionApiMapping.CommandMeta;
|
import org.apache.solr.client.solrj.request.CollectionApiMapping.CommandMeta;
|
||||||
import org.apache.solr.client.solrj.request.CollectionApiMapping.Meta;
|
import org.apache.solr.client.solrj.request.CollectionApiMapping.Meta;
|
||||||
import org.apache.solr.client.solrj.request.CollectionApiMapping.V2EndPoint;
|
import org.apache.solr.client.solrj.request.CollectionApiMapping.V2EndPoint;
|
||||||
|
import org.apache.solr.common.Callable;
|
||||||
|
import org.apache.solr.common.SolrException;
|
||||||
|
import org.apache.solr.common.cloud.ClusterProperties;
|
||||||
|
import org.apache.solr.common.util.CommandOperation;
|
||||||
import org.apache.solr.handler.admin.CollectionsHandler.CollectionOperation;
|
import org.apache.solr.handler.admin.CollectionsHandler.CollectionOperation;
|
||||||
import org.apache.solr.request.SolrQueryRequest;
|
import org.apache.solr.request.SolrQueryRequest;
|
||||||
import org.apache.solr.response.SolrQueryResponse;
|
import org.apache.solr.response.SolrQueryResponse;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
public class CollectionHandlerApi extends BaseHandlerApiSupport {
|
public class CollectionHandlerApi extends BaseHandlerApiSupport {
|
||||||
|
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
|
||||||
|
|
||||||
final CollectionsHandler handler;
|
final CollectionsHandler handler;
|
||||||
static Collection<ApiCommand> apiCommands = createCollMapping();
|
static Collection<ApiCommand> apiCommands = createCollMapping();
|
||||||
|
|
||||||
|
@ -55,26 +64,55 @@ public class CollectionHandlerApi extends BaseHandlerApiSupport {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
result.put(Meta.GET_NODES, new ApiCommand() {
|
//The following APIs have only V2 implementations
|
||||||
@Override
|
addApi(result, Meta.GET_NODES, params -> params.rsp.add("nodes", ((CollectionHandlerApi) params.apiHandler).handler.coreContainer.getZkController().getClusterState().getLiveNodes()));
|
||||||
public CommandMeta meta() {
|
addApi(result, Meta.SET_CLUSTER_PROPERTY_OBJ, params -> {
|
||||||
return Meta.GET_NODES;
|
List<CommandOperation> commands = params.req.getCommands(true);
|
||||||
}
|
if (commands == null || commands.isEmpty()) throw new RuntimeException("Empty commands");
|
||||||
|
ClusterProperties clusterProperties = new ClusterProperties(((CollectionHandlerApi) params.apiHandler).handler.coreContainer.getZkController().getZkClient());
|
||||||
|
|
||||||
@Override
|
try {
|
||||||
public void invoke(SolrQueryRequest req, SolrQueryResponse rsp, BaseHandlerApiSupport apiHandler) throws Exception {
|
clusterProperties.setClusterProperties(commands.get(0).getDataMap());
|
||||||
rsp.add("nodes", ((CollectionHandlerApi) apiHandler).handler.coreContainer.getZkController().getClusterState().getLiveNodes());
|
} catch (Exception e) {
|
||||||
|
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error in API", e);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
for (Meta meta : Meta.values()) {
|
for (Meta meta : Meta.values()) {
|
||||||
if(result.get(meta) == null){
|
if (result.get(meta) == null) {
|
||||||
throw new RuntimeException("No implementation for "+ meta.name());
|
log.error("ERROR_INIT. No corresponding API implementation for : " + meta.commandName);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return result.values();
|
return result.values();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private static void addApi(Map<Meta, ApiCommand> result, Meta metaInfo, Callable<ApiParams> fun) {
|
||||||
|
result.put(metaInfo, new ApiCommand() {
|
||||||
|
@Override
|
||||||
|
public CommandMeta meta() {
|
||||||
|
return metaInfo;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void invoke(SolrQueryRequest req, SolrQueryResponse rsp, BaseHandlerApiSupport apiHandler) throws Exception {
|
||||||
|
fun.call(new ApiParams(req, rsp, apiHandler));
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
static class ApiParams {
|
||||||
|
final SolrQueryRequest req;
|
||||||
|
final SolrQueryResponse rsp;
|
||||||
|
final BaseHandlerApiSupport apiHandler;
|
||||||
|
|
||||||
|
ApiParams(SolrQueryRequest req, SolrQueryResponse rsp, BaseHandlerApiSupport apiHandler) {
|
||||||
|
this.req = req;
|
||||||
|
this.rsp = rsp;
|
||||||
|
this.apiHandler = apiHandler;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
public CollectionHandlerApi(CollectionsHandler handler) {
|
public CollectionHandlerApi(CollectionsHandler handler) {
|
||||||
this.handler = handler;
|
this.handler = handler;
|
||||||
}
|
}
|
||||||
|
|
|
@ -33,6 +33,7 @@ import java.util.Set;
|
||||||
import java.util.concurrent.TimeUnit;
|
import java.util.concurrent.TimeUnit;
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
|
import com.google.common.collect.ImmutableList;
|
||||||
import com.google.common.collect.ImmutableSet;
|
import com.google.common.collect.ImmutableSet;
|
||||||
import org.apache.commons.io.IOUtils;
|
import org.apache.commons.io.IOUtils;
|
||||||
import org.apache.commons.lang.StringUtils;
|
import org.apache.commons.lang.StringUtils;
|
||||||
|
@ -122,6 +123,7 @@ import static org.apache.solr.common.cloud.DocCollection.RULE;
|
||||||
import static org.apache.solr.common.cloud.DocCollection.SNITCH;
|
import static org.apache.solr.common.cloud.DocCollection.SNITCH;
|
||||||
import static org.apache.solr.common.cloud.DocCollection.STATE_FORMAT;
|
import static org.apache.solr.common.cloud.DocCollection.STATE_FORMAT;
|
||||||
import static org.apache.solr.common.cloud.ZkStateReader.AUTO_ADD_REPLICAS;
|
import static org.apache.solr.common.cloud.ZkStateReader.AUTO_ADD_REPLICAS;
|
||||||
|
import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_DEF;
|
||||||
import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP;
|
import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP;
|
||||||
import static org.apache.solr.common.cloud.ZkStateReader.MAX_SHARDS_PER_NODE;
|
import static org.apache.solr.common.cloud.ZkStateReader.MAX_SHARDS_PER_NODE;
|
||||||
import static org.apache.solr.common.cloud.ZkStateReader.NRT_REPLICAS;
|
import static org.apache.solr.common.cloud.ZkStateReader.NRT_REPLICAS;
|
||||||
|
@ -204,6 +206,12 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission
|
||||||
return this.coreContainer;
|
return this.coreContainer;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
protected void copyFromClusterProp(Map<String, Object> props, String prop) {
|
||||||
|
if (props.get(prop) != null) return;//if it's already specified , return
|
||||||
|
Object defVal = coreContainer.getZkController().getZkStateReader().getClusterProperty(ImmutableList.of(COLLECTION_DEF, prop), null);
|
||||||
|
if (defVal != null) props.put(prop, String.valueOf(defVal));
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {
|
public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {
|
||||||
// Make sure the cores is enabled
|
// Make sure the cores is enabled
|
||||||
|
@ -490,6 +498,9 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission
|
||||||
createSysConfigSet(h.coreContainer);
|
createSysConfigSet(h.coreContainer);
|
||||||
|
|
||||||
}
|
}
|
||||||
|
if (shardsParam == null) h.copyFromClusterProp(props, NUM_SLICES);
|
||||||
|
for (String prop : ImmutableSet.of(NRT_REPLICAS, PULL_REPLICAS, TLOG_REPLICAS))
|
||||||
|
h.copyFromClusterProp(props, prop);
|
||||||
copyPropertiesWithPrefix(req.getParams(), props, COLL_PROP_PREFIX);
|
copyPropertiesWithPrefix(req.getParams(), props, COLL_PROP_PREFIX);
|
||||||
return copyPropertiesWithPrefix(req.getParams(), props, "router.");
|
return copyPropertiesWithPrefix(req.getParams(), props, "router.");
|
||||||
|
|
||||||
|
|
|
@ -27,15 +27,19 @@ import java.util.Objects;
|
||||||
import java.util.Optional;
|
import java.util.Optional;
|
||||||
import java.util.concurrent.TimeUnit;
|
import java.util.concurrent.TimeUnit;
|
||||||
|
|
||||||
|
import com.google.common.collect.ImmutableList;
|
||||||
import org.apache.lucene.util.LuceneTestCase;
|
import org.apache.lucene.util.LuceneTestCase;
|
||||||
import org.apache.lucene.util.TestUtil;
|
import org.apache.lucene.util.TestUtil;
|
||||||
|
import org.apache.solr.client.solrj.SolrRequest;
|
||||||
import org.apache.solr.client.solrj.SolrServerException;
|
import org.apache.solr.client.solrj.SolrServerException;
|
||||||
import org.apache.solr.client.solrj.impl.HttpSolrClient;
|
import org.apache.solr.client.solrj.impl.HttpSolrClient;
|
||||||
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
|
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
|
||||||
import org.apache.solr.client.solrj.request.CoreAdminRequest;
|
import org.apache.solr.client.solrj.request.CoreAdminRequest;
|
||||||
import org.apache.solr.client.solrj.request.CoreStatus;
|
import org.apache.solr.client.solrj.request.CoreStatus;
|
||||||
|
import org.apache.solr.client.solrj.request.V2Request;
|
||||||
import org.apache.solr.client.solrj.response.CollectionAdminResponse;
|
import org.apache.solr.client.solrj.response.CollectionAdminResponse;
|
||||||
import org.apache.solr.client.solrj.response.CoreAdminResponse;
|
import org.apache.solr.client.solrj.response.CoreAdminResponse;
|
||||||
|
import org.apache.solr.client.solrj.response.V2Response;
|
||||||
import org.apache.solr.common.cloud.ClusterProperties;
|
import org.apache.solr.common.cloud.ClusterProperties;
|
||||||
import org.apache.solr.common.cloud.DocCollection;
|
import org.apache.solr.common.cloud.DocCollection;
|
||||||
import org.apache.solr.common.cloud.Replica;
|
import org.apache.solr.common.cloud.Replica;
|
||||||
|
@ -50,6 +54,10 @@ import org.apache.zookeeper.KeeperException;
|
||||||
import org.junit.BeforeClass;
|
import org.junit.BeforeClass;
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
|
|
||||||
|
import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_DEF;
|
||||||
|
import static org.apache.solr.common.cloud.ZkStateReader.NRT_REPLICAS;
|
||||||
|
import static org.apache.solr.common.cloud.ZkStateReader.NUM_SHARDS_PROP;
|
||||||
|
|
||||||
@LuceneTestCase.Slow
|
@LuceneTestCase.Slow
|
||||||
public class CollectionsAPISolrJTest extends SolrCloudTestCase {
|
public class CollectionsAPISolrJTest extends SolrCloudTestCase {
|
||||||
|
|
||||||
|
@ -91,6 +99,49 @@ public class CollectionsAPISolrJTest extends SolrCloudTestCase {
|
||||||
waitForState("Expected " + collectionName + " to disappear from cluster state", collectionName, (n, c) -> c == null);
|
waitForState("Expected " + collectionName + " to disappear from cluster state", collectionName, (n, c) -> c == null);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testCreateCollWithDefaultClusterProperties() throws Exception {
|
||||||
|
String COLL_NAME = "CollWithDefaultClusterProperties";
|
||||||
|
try {
|
||||||
|
V2Response rsp = new V2Request.Builder("/cluster")
|
||||||
|
.withMethod(SolrRequest.METHOD.POST)
|
||||||
|
.withPayload("{set-obj-property:{collectionDefaults:{numShards : 2 , nrtReplicas : 2}}}")
|
||||||
|
.build()
|
||||||
|
.process(cluster.getSolrClient());
|
||||||
|
|
||||||
|
for (int i = 0; i < 10; i++) {
|
||||||
|
Map m = cluster.getSolrClient().getZkStateReader().getClusterProperty(COLLECTION_DEF, null);
|
||||||
|
if (m != null) break;
|
||||||
|
Thread.sleep(10);
|
||||||
|
}
|
||||||
|
Object clusterProperty = cluster.getSolrClient().getZkStateReader().getClusterProperty(ImmutableList.of(COLLECTION_DEF, NUM_SHARDS_PROP), null);
|
||||||
|
assertEquals("2", String.valueOf(clusterProperty));
|
||||||
|
clusterProperty = cluster.getSolrClient().getZkStateReader().getClusterProperty(ImmutableList.of(COLLECTION_DEF, NRT_REPLICAS), null);
|
||||||
|
assertEquals("2", String.valueOf(clusterProperty));
|
||||||
|
CollectionAdminResponse response = CollectionAdminRequest
|
||||||
|
.createCollection(COLL_NAME, "conf", null, null, null, null)
|
||||||
|
.process(cluster.getSolrClient());
|
||||||
|
assertEquals(0, response.getStatus());
|
||||||
|
assertTrue(response.isSuccess());
|
||||||
|
|
||||||
|
DocCollection coll = cluster.getSolrClient().getClusterStateProvider().getClusterState().getCollection(COLL_NAME);
|
||||||
|
Map<String, Slice> slices = coll.getSlicesMap();
|
||||||
|
assertEquals(2, slices.size());
|
||||||
|
for (Slice slice : slices.values()) {
|
||||||
|
assertEquals(2, slice.getReplicas().size());
|
||||||
|
}
|
||||||
|
CollectionAdminRequest.deleteCollection(COLL_NAME).process(cluster.getSolrClient());
|
||||||
|
} finally {
|
||||||
|
V2Response rsp = new V2Request.Builder("/cluster")
|
||||||
|
.withMethod(SolrRequest.METHOD.POST)
|
||||||
|
.withPayload("{set-obj-property:{collectionDefaults: null}}")
|
||||||
|
.build()
|
||||||
|
.process(cluster.getSolrClient());
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testCreateAndDeleteCollection() throws Exception {
|
public void testCreateAndDeleteCollection() throws Exception {
|
||||||
String collectionName = "solrj_test";
|
String collectionName = "solrj_test";
|
||||||
|
|
|
@ -278,6 +278,11 @@ public class TestCollectionAPIs extends SolrTestCaseJ4 {
|
||||||
MockCollectionsHandler() {
|
MockCollectionsHandler() {
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected void copyFromClusterProp(Map<String, Object> props, String prop) {
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
void invokeAction(SolrQueryRequest req, SolrQueryResponse rsp,
|
void invokeAction(SolrQueryRequest req, SolrQueryResponse rsp,
|
||||||
CoreContainer cores,
|
CoreContainer cores,
|
||||||
|
|
|
@ -26,6 +26,7 @@ import java.util.Iterator;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
|
||||||
|
import com.google.common.collect.ImmutableList;
|
||||||
import org.apache.solr.SolrTestCaseJ4;
|
import org.apache.solr.SolrTestCaseJ4;
|
||||||
import org.apache.solr.common.MapWriter;
|
import org.apache.solr.common.MapWriter;
|
||||||
import org.apache.solr.common.util.CommandOperation;
|
import org.apache.solr.common.util.CommandOperation;
|
||||||
|
@ -39,6 +40,9 @@ import org.apache.solr.common.util.Utils;
|
||||||
import org.junit.Assert;
|
import org.junit.Assert;
|
||||||
|
|
||||||
import static java.nio.charset.StandardCharsets.UTF_8;
|
import static java.nio.charset.StandardCharsets.UTF_8;
|
||||||
|
import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_DEF;
|
||||||
|
import static org.apache.solr.common.cloud.ZkStateReader.NRT_REPLICAS;
|
||||||
|
import static org.apache.solr.common.cloud.ZkStateReader.NUM_SHARDS_PROP;
|
||||||
import static org.apache.solr.common.util.Utils.fromJSONString;
|
import static org.apache.solr.common.util.Utils.fromJSONString;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -248,4 +252,24 @@ public class TestUtils extends SolrTestCaseJ4 {
|
||||||
assertEquals("x-update", Utils.getObjectByPath(m,false, "authorization/permissions[1]/name"));
|
assertEquals("x-update", Utils.getObjectByPath(m,false, "authorization/permissions[1]/name"));
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public void testMergeJson() {
|
||||||
|
Map<String, Object> sink = (Map<String, Object>) Utils.fromJSONString("{k2:v2, k1: {a:b, p:r, k21:{xx:yy}}}");
|
||||||
|
assertTrue(Utils.mergeJson(sink, (Map<String, Object>) Utils.fromJSONString("k1:{a:c, e:f, p :null, k11:{a1:b1}, k21:{pp : qq}}")));
|
||||||
|
|
||||||
|
assertEquals("v2", Utils.getObjectByPath(sink, true, "k2"));
|
||||||
|
assertEquals("c", Utils.getObjectByPath(sink, true, "k1/a"));
|
||||||
|
assertEquals("yy", Utils.getObjectByPath(sink, true, "k1/k21/xx"));
|
||||||
|
assertEquals("qq", Utils.getObjectByPath(sink, true, "k1/k21/pp"));
|
||||||
|
assertEquals("f", Utils.getObjectByPath(sink, true, "k1/e"));
|
||||||
|
assertEquals("b1", Utils.getObjectByPath(sink, true, "k1/k11/a1"));
|
||||||
|
|
||||||
|
sink = new HashMap<>();
|
||||||
|
sink.put("legacyCloud", "false");
|
||||||
|
assertTrue(Utils.mergeJson(sink, (Map<String, Object>) Utils.fromJSONString("collectionDefaults:{numShards:3 , nrtReplicas:2}")));
|
||||||
|
assertEquals(3l, Utils.getObjectByPath(sink, true, ImmutableList.of(COLLECTION_DEF, NUM_SHARDS_PROP)));
|
||||||
|
assertEquals(2l, Utils.getObjectByPath(sink, true, ImmutableList.of(COLLECTION_DEF, NRT_REPLICAS)));
|
||||||
|
|
||||||
|
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -320,7 +320,7 @@ public abstract class CollectionAdminRequest<T extends CollectionAdminResponse>
|
||||||
* @param numTlogReplicas the number of {@link org.apache.solr.common.cloud.Replica.Type#TLOG} replicas
|
* @param numTlogReplicas the number of {@link org.apache.solr.common.cloud.Replica.Type#TLOG} replicas
|
||||||
* @param numPullReplicas the number of {@link org.apache.solr.common.cloud.Replica.Type#PULL} replicas
|
* @param numPullReplicas the number of {@link org.apache.solr.common.cloud.Replica.Type#PULL} replicas
|
||||||
*/
|
*/
|
||||||
public static Create createCollection(String collection, String config, int numShards, int numNrtReplicas, int numTlogReplicas, int numPullReplicas) {
|
public static Create createCollection(String collection, String config, Integer numShards, Integer numNrtReplicas, Integer numTlogReplicas, Integer numPullReplicas) {
|
||||||
return new Create(collection, config, numShards, numNrtReplicas, numTlogReplicas, numPullReplicas);
|
return new Create(collection, config, numShards, numNrtReplicas, numTlogReplicas, numPullReplicas);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -203,6 +203,10 @@ public class CollectionApiMapping {
|
||||||
POST,
|
POST,
|
||||||
CLUSTERPROP,
|
CLUSTERPROP,
|
||||||
"set-property",null),
|
"set-property",null),
|
||||||
|
SET_CLUSTER_PROPERTY_OBJ(CLUSTER_CMD,
|
||||||
|
POST,
|
||||||
|
null,
|
||||||
|
"set-obj-property", null),
|
||||||
UTILIZE_NODE(CLUSTER_CMD,
|
UTILIZE_NODE(CLUSTER_CMD,
|
||||||
POST,
|
POST,
|
||||||
UTILIZENODE,
|
UTILIZENODE,
|
||||||
|
|
|
@ -18,6 +18,7 @@
|
||||||
package org.apache.solr.common.cloud;
|
package org.apache.solr.common.cloud;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
|
import java.lang.invoke.MethodHandles;
|
||||||
import java.util.Collections;
|
import java.util.Collections;
|
||||||
import java.util.LinkedHashMap;
|
import java.util.LinkedHashMap;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
@ -27,6 +28,8 @@ import org.apache.solr.common.util.Utils;
|
||||||
import org.apache.zookeeper.CreateMode;
|
import org.apache.zookeeper.CreateMode;
|
||||||
import org.apache.zookeeper.KeeperException;
|
import org.apache.zookeeper.KeeperException;
|
||||||
import org.apache.zookeeper.data.Stat;
|
import org.apache.zookeeper.data.Stat;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Interact with solr cluster properties
|
* Interact with solr cluster properties
|
||||||
|
@ -36,6 +39,8 @@ import org.apache.zookeeper.data.Stat;
|
||||||
* {@link ZkStateReader#getClusterProperty(String, Object)}
|
* {@link ZkStateReader#getClusterProperty(String, Object)}
|
||||||
*/
|
*/
|
||||||
public class ClusterProperties {
|
public class ClusterProperties {
|
||||||
|
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
|
||||||
|
|
||||||
|
|
||||||
private final SolrZkClient client;
|
private final SolrZkClient client;
|
||||||
|
|
||||||
|
@ -48,7 +53,7 @@ public class ClusterProperties {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Read the value of a cluster property, returning a default if it is not set
|
* Read the value of a cluster property, returning a default if it is not set
|
||||||
* @param key the property name
|
* @param key the property name or the full path to the property.
|
||||||
* @param defaultValue the default value
|
* @param defaultValue the default value
|
||||||
* @param <T> the type of the property
|
* @param <T> the type of the property
|
||||||
* @return the property value
|
* @return the property value
|
||||||
|
@ -56,7 +61,7 @@ public class ClusterProperties {
|
||||||
*/
|
*/
|
||||||
@SuppressWarnings("unchecked")
|
@SuppressWarnings("unchecked")
|
||||||
public <T> T getClusterProperty(String key, T defaultValue) throws IOException {
|
public <T> T getClusterProperty(String key, T defaultValue) throws IOException {
|
||||||
T value = (T) getClusterProperties().get(key);
|
T value = (T) Utils.getObjectByPath(getClusterProperties(), false, key);
|
||||||
if (value == null)
|
if (value == null)
|
||||||
return defaultValue;
|
return defaultValue;
|
||||||
return value;
|
return value;
|
||||||
|
@ -77,6 +82,15 @@ public class ClusterProperties {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public void setClusterProperties(Map<String, Object> properties) throws IOException, KeeperException, InterruptedException {
|
||||||
|
client.atomicUpdate(ZkStateReader.CLUSTER_PROPS, zkData -> {
|
||||||
|
if (zkData == null) return Utils.toJSON(properties);
|
||||||
|
Map<String, Object> zkJson = (Map<String, Object>) Utils.fromJSON(zkData);
|
||||||
|
boolean modified = Utils.mergeJson(zkJson, properties);
|
||||||
|
return modified ? Utils.toJSON(zkJson) : null;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* This method sets a cluster property.
|
* This method sets a cluster property.
|
||||||
*
|
*
|
||||||
|
|
|
@ -34,6 +34,7 @@ import java.nio.file.Path;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.concurrent.ExecutorService;
|
import java.util.concurrent.ExecutorService;
|
||||||
import java.util.concurrent.RejectedExecutionException;
|
import java.util.concurrent.RejectedExecutionException;
|
||||||
|
import java.util.function.Function;
|
||||||
import java.util.function.Predicate;
|
import java.util.function.Predicate;
|
||||||
import java.util.regex.Pattern;
|
import java.util.regex.Pattern;
|
||||||
|
|
||||||
|
@ -355,6 +356,38 @@ public class SolrZkClient implements Closeable {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public void atomicUpdate(String path, Function<byte[], byte[]> editor) throws KeeperException, InterruptedException {
|
||||||
|
for (; ; ) {
|
||||||
|
byte[] modified = null;
|
||||||
|
byte[] zkData = null;
|
||||||
|
Stat s = new Stat();
|
||||||
|
try {
|
||||||
|
if (exists(path, true)) {
|
||||||
|
zkData = getData(path, null, s, true);
|
||||||
|
modified = editor.apply(zkData);
|
||||||
|
if (modified == null) {
|
||||||
|
//no change , no need to persist
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
setData(path, modified, s.getVersion(), true);
|
||||||
|
break;
|
||||||
|
} else {
|
||||||
|
modified = editor.apply(null);
|
||||||
|
if (modified == null) {
|
||||||
|
//no change , no need to persist
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
create(path, modified, CreateMode.PERSISTENT, true);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
} catch (KeeperException.BadVersionException | KeeperException.NodeExistsException e) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Returns path of created node
|
* Returns path of created node
|
||||||
*/
|
*/
|
||||||
|
|
|
@ -127,6 +127,7 @@ public class ZkStateReader implements Closeable {
|
||||||
public final static String CONFIGNAME_PROP="configName";
|
public final static String CONFIGNAME_PROP="configName";
|
||||||
|
|
||||||
public static final String LEGACY_CLOUD = "legacyCloud";
|
public static final String LEGACY_CLOUD = "legacyCloud";
|
||||||
|
public static final String COLLECTION_DEF = "collectionDefaults";
|
||||||
|
|
||||||
public static final String URL_SCHEME = "urlScheme";
|
public static final String URL_SCHEME = "urlScheme";
|
||||||
|
|
||||||
|
@ -954,7 +955,14 @@ public class ZkStateReader implements Closeable {
|
||||||
*/
|
*/
|
||||||
@SuppressWarnings("unchecked")
|
@SuppressWarnings("unchecked")
|
||||||
public <T> T getClusterProperty(String key, T defaultValue) {
|
public <T> T getClusterProperty(String key, T defaultValue) {
|
||||||
T value = (T) clusterProperties.get(key);
|
T value = (T) Utils.getObjectByPath( clusterProperties, false, key);
|
||||||
|
if (value == null)
|
||||||
|
return defaultValue;
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
|
||||||
|
public <T> T getClusterProperty(List<String> keyPath, T defaultValue) {
|
||||||
|
T value = (T) Utils.getObjectByPath( clusterProperties, false, keyPath);
|
||||||
if (value == null)
|
if (value == null)
|
||||||
return defaultValue;
|
return defaultValue;
|
||||||
return value;
|
return value;
|
||||||
|
|
|
@ -74,9 +74,10 @@ public class JsonSchemaValidator {
|
||||||
return errs.isEmpty() ? null : errs;
|
return errs.isEmpty() ? null : errs;
|
||||||
}
|
}
|
||||||
|
|
||||||
boolean validate(Object data, List<String> errs){
|
boolean validate(Object data, List<String> errs) {
|
||||||
|
if (data == null) return true;
|
||||||
for (Validator validator : validators) {
|
for (Validator validator : validators) {
|
||||||
if(!validator.validate(data, errs)) {
|
if (!validator.validate(data, errs)) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -456,6 +456,48 @@ public class Utils {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**Applies one json over other. The 'input' is applied over the sink
|
||||||
|
* The values in input isapplied over the values in 'sink' . If a value is 'null'
|
||||||
|
* that value is removed from sink
|
||||||
|
*
|
||||||
|
* @param sink the original json object to start with. Ensure that this Map is mutable
|
||||||
|
* @param input the json with new values
|
||||||
|
* @return whether there was any change made to sink or not.
|
||||||
|
*/
|
||||||
|
|
||||||
|
public static boolean mergeJson(Map<String, Object> sink, Map<String, Object> input) {
|
||||||
|
boolean isModified = false;
|
||||||
|
for (Map.Entry<String, Object> e : input.entrySet()) {
|
||||||
|
if (sink.get(e.getKey()) != null) {
|
||||||
|
Object sinkVal = sink.get(e.getKey());
|
||||||
|
if (e.getValue() == null) {
|
||||||
|
sink.remove(e.getKey());
|
||||||
|
isModified = true;
|
||||||
|
} else {
|
||||||
|
if (e.getValue() instanceof Map) {
|
||||||
|
Map<String, Object> mapInputVal = (Map<String, Object>) e.getValue();
|
||||||
|
if (sinkVal instanceof Map) {
|
||||||
|
if (mergeJson((Map<String, Object>) sinkVal, mapInputVal)) isModified = true;
|
||||||
|
} else {
|
||||||
|
sink.put(e.getKey(), mapInputVal);
|
||||||
|
isModified = true;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
sink.put(e.getKey(), e.getValue());
|
||||||
|
isModified = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
} else if (e.getValue() != null) {
|
||||||
|
sink.put(e.getKey(), e.getValue());
|
||||||
|
isModified = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
return isModified;
|
||||||
|
}
|
||||||
|
|
||||||
public static String getBaseUrlForNodeName(final String nodeName, String urlScheme) {
|
public static String getBaseUrlForNodeName(final String nodeName, String urlScheme) {
|
||||||
final int _offset = nodeName.indexOf("_");
|
final int _offset = nodeName.indexOf("_");
|
||||||
if (_offset < 0) {
|
if (_offset < 0) {
|
||||||
|
|
|
@ -70,6 +70,49 @@
|
||||||
"val"
|
"val"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
|
"set-obj-property": {
|
||||||
|
"type": "object",
|
||||||
|
"documentation": "https://lucene.apache.org/solr/guide/collections-api.html#clusterprop",
|
||||||
|
"description": "Add, edit, or delete a cluster-wide property.",
|
||||||
|
"properties": {
|
||||||
|
"legacyCloud": {
|
||||||
|
"type": "boolean"
|
||||||
|
},
|
||||||
|
"urlScheme": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"autoAddReplicas": {
|
||||||
|
"type": "boolean"
|
||||||
|
},
|
||||||
|
"maxCoresPerNode": {
|
||||||
|
"type": "boolean"
|
||||||
|
},
|
||||||
|
"location": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"collectionDefaults": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"numShards": {
|
||||||
|
"type": "integer",
|
||||||
|
"description": "Default no:of shards for a collection"
|
||||||
|
},
|
||||||
|
"tlogReplicas": {
|
||||||
|
"type": "integer",
|
||||||
|
"description": "Default no:of TLOG replicas"
|
||||||
|
},
|
||||||
|
"pullReplicas": {
|
||||||
|
"type": "integer",
|
||||||
|
"description": "Default no:of PULL replicas"
|
||||||
|
},
|
||||||
|
"nrtReplicas": {
|
||||||
|
"type": "integer",
|
||||||
|
"description": "Default no:of NRT replicas"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
"utilize-node": {
|
"utilize-node": {
|
||||||
"type": "object",
|
"type": "object",
|
||||||
"documentation": "https://lucene.apache.org/solr/guide/collections-api.html#utilizenode",
|
"documentation": "https://lucene.apache.org/solr/guide/collections-api.html#utilizenode",
|
||||||
|
|
|
@ -40,6 +40,7 @@ public class JsonValidatorTest extends SolrTestCaseJ4 {
|
||||||
checkSchema("core.config.Commands");
|
checkSchema("core.config.Commands");
|
||||||
checkSchema("core.SchemaEdit");
|
checkSchema("core.SchemaEdit");
|
||||||
checkSchema("cluster.configs.Commands");
|
checkSchema("cluster.configs.Commands");
|
||||||
|
checkSchema("cluster.Commands");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@ -176,6 +177,13 @@ public class JsonValidatorTest extends SolrTestCaseJ4 {
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public void testNullObjectValue() {
|
||||||
|
ValidatingJsonMap spec = Utils.getSpec("cluster.Commands").getSpec();
|
||||||
|
JsonSchemaValidator validator = new JsonSchemaValidator((Map) Utils.getObjectByPath(spec, false, "/commands/set-obj-property"));
|
||||||
|
List<String> object = validator.validateJson(Utils.fromJSONString("{collectionDefaults: null}"));
|
||||||
|
assertNull(object);
|
||||||
|
}
|
||||||
|
|
||||||
private void checkSchema(String name) {
|
private void checkSchema(String name) {
|
||||||
ValidatingJsonMap spec = Utils.getSpec(name).getSpec();
|
ValidatingJsonMap spec = Utils.getSpec(name).getSpec();
|
||||||
Map commands = (Map) spec.get("commands");
|
Map commands = (Map) spec.get("commands");
|
||||||
|
|
Loading…
Reference in New Issue