Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-8146: refactored the replica rules classes so that they can be accessed from SolrJ

This commit is contained in:
Noble Paul 2016-09-29 19:37:35 +05:30
parent 64ab29a796
commit 3ab22f6e3a
12 changed files with 374 additions and 234 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

145
solr/core/src/java/org/apache/solr/cloud/rule/ImplicitSnitch.java
View File

@ -14,91 +14,25 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.solr.cloud.rule; package org.apache.solr.cloud.rule;
import java.io.IOException; import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.net.InetAddress;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap; import java.util.HashMap;
import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.core.CoreContainer; import org.apache.solr.core.CoreContainer;
import org.apache.solr.handler.admin.CoreAdminHandler; import org.apache.solr.handler.admin.CoreAdminHandler;
import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.request.SolrQueryRequest;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class ImplicitSnitch extends Snitch implements CoreAdminHandler.Invocable { import static org.apache.solr.common.cloud.rule.ImplicitSnitch.CORES;
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); import static org.apache.solr.common.cloud.rule.ImplicitSnitch.DISK;
import static org.apache.solr.common.cloud.rule.ImplicitSnitch.SYSPROP;
public static final Pattern hostAndPortPattern = Pattern.compile("(?:https?://)?([^:]+):(\\d+)"); //this is the server-side component which provides the tag values
public class ImplicitSnitch implements CoreAdminHandler.Invocable {
//well known tags
public static final String NODE = "node";
public static final String PORT = "port";
public static final String HOST = "host";
public static final String CORES = "cores";
public static final String DISK = "freedisk";
public static final String ROLE = "role";
public static final String SYSPROP = "sysprop.";
public static final List<String> IP_SNITCHES = ImmutableList.of("ip_1", "ip_2", "ip_3", "ip_4");
public static final Set<String> tags = ImmutableSet.<String>builder().add(NODE, PORT, HOST, CORES, DISK, ROLE).addAll(IP_SNITCHES).build();
@Override
public void getTags(String solrNode, Set<String> requestedTags, SnitchContext ctx) {
if (requestedTags.contains(NODE)) ctx.getTags().put(NODE, solrNode);
if (requestedTags.contains(HOST)) {
Matcher hostAndPortMatcher = hostAndPortPattern.matcher(solrNode);
if (hostAndPortMatcher.find()) ctx.getTags().put(HOST, hostAndPortMatcher.group(1));
}
if (requestedTags.contains(PORT)) {
Matcher hostAndPortMatcher = hostAndPortPattern.matcher(solrNode);
if (hostAndPortMatcher.find()) ctx.getTags().put(PORT, hostAndPortMatcher.group(2));
}
if (requestedTags.contains(ROLE)) fillRole(solrNode, ctx);
addIpTags(solrNode, requestedTags, ctx);
ModifiableSolrParams params = new ModifiableSolrParams();
if (requestedTags.contains(CORES)) params.add(CORES, "1");
if (requestedTags.contains(DISK)) params.add(DISK, "1");
for (String tag : requestedTags) {
if (tag.startsWith(SYSPROP)) params.add(SYSPROP, tag.substring(SYSPROP.length()));
}
if (params.size() > 0) ctx.invokeRemote(solrNode, params, ImplicitSnitch.class.getName(), null);
}
private void fillRole(String solrNode, SnitchContext ctx) {
Map roles = (Map) ctx.retrieve(ZkStateReader.ROLES); // we don't want to hit the ZK for each node
// so cache and reuse
if(roles == null) roles = ctx.getZkJson(ZkStateReader.ROLES);
ctx.store(ZkStateReader.ROLES, roles == null ? Collections.emptyMap() : roles);
if (roles != null) {
for (Object o : roles.entrySet()) {
Map.Entry e = (Map.Entry) o;
if (e.getValue() instanceof List) {
if(((List) e.getValue()).contains(solrNode)) {
ctx.getTags().put(ROLE, e.getKey());
break;
}
}
}
}
}
static long getUsableSpaceInGB(Path path) throws IOException { static long getUsableSpaceInGB(Path path) throws IOException {
long space = Files.getFileStore(path).getUsableSpace(); long space = Files.getFileStore(path).getUsableSpace();
@ -106,6 +40,7 @@ public class ImplicitSnitch extends Snitch implements CoreAdminHandler.Invocable
return spaceInGB; return spaceInGB;
} }
@Override
public Map<String, Object> invoke(SolrQueryRequest req) { public Map<String, Object> invoke(SolrQueryRequest req) {
Map<String, Object> result = new HashMap<>(); Map<String, Object> result = new HashMap<>();
CoreContainer cc = (CoreContainer) req.getContext().get(CoreContainer.class.getName()); CoreContainer cc = (CoreContainer) req.getContext().get(CoreContainer.class.getName());
@ -127,70 +62,4 @@ public class ImplicitSnitch extends Snitch implements CoreAdminHandler.Invocable
return result; return result;
} }
private static final String HOST_FRAG_SEPARATOR_REGEX = "\\.";
@Override
public boolean isKnownTag(String tag) {
return tags.contains(tag) ||
tag.startsWith(SYSPROP);
}
private void addIpTags(String solrNode, Set<String> requestedTags, SnitchContext context) {
List<String> requestedHostTags = new ArrayList<>();
for (String tag : requestedTags) {
if (IP_SNITCHES.contains(tag)) {
requestedHostTags.add(tag);
}
}
if (requestedHostTags.isEmpty()) {
return;
}
String[] ipFragments = getIpFragments(solrNode);
if (ipFragments == null) {
return;
}
int ipSnitchCount = IP_SNITCHES.size();
for (int i = 0; i < ipSnitchCount; i++) {
String currentTagValue = ipFragments[i];
String currentTagKey = IP_SNITCHES.get(ipSnitchCount - i - 1);
if (requestedHostTags.contains(currentTagKey)) {
context.getTags().put(currentTagKey, currentTagValue);
}
}
}
private String[] getIpFragments(String solrNode) {
Matcher hostAndPortMatcher = hostAndPortPattern.matcher(solrNode);
if (hostAndPortMatcher.find()) {
String host = hostAndPortMatcher.group(1);
if (host != null) {
String ip = getHostIp(host);
if (ip != null) {
return ip.split(HOST_FRAG_SEPARATOR_REGEX); //IPv6 support will be provided by SOLR-8523
}
}
}
log.warn("Failed to match host IP address from node URL [{}] using regex [{}]", solrNode, hostAndPortPattern.pattern());
return null;
}
protected String getHostIp(String host) {
try {
InetAddress address = InetAddress.getByName(host);
return address.getHostAddress();
} catch (Exception e) {
log.warn("Failed to get IP address from host [{}], with exception [{}] ", host, e);
return null;
}
}
} }

17
solr/core/src/java/org/apache/solr/cloud/rule/ReplicaAssigner.java
View File

@ -36,6 +36,9 @@ import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.DocCollection; import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.Replica; import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.Slice; import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.rule.ImplicitSnitch;
import org.apache.solr.common.cloud.rule.Snitch;
import org.apache.solr.common.cloud.rule.SnitchContext;
import org.apache.solr.common.util.StrUtils; import org.apache.solr.common.util.StrUtils;
import org.apache.solr.common.util.Utils; import org.apache.solr.common.util.Utils;
import org.apache.solr.core.CoreContainer; import org.apache.solr.core.CoreContainer;
@ -360,10 +363,7 @@ public class ReplicaAssigner {
return myTags; return myTags;
} }
@Override
public CoreContainer getCoreContainer() {
return cc;
}
} }
/** /**
@ -399,7 +399,7 @@ public class ReplicaAssigner {
//now use the Snitch to get the tags //now use the Snitch to get the tags
for (SnitchInfoImpl info : snitches.values()) { for (SnitchInfoImpl info : snitches.values()) {
if (!info.myTags.isEmpty()) { if (!info.myTags.isEmpty()) {
SnitchContext context = getSnitchCtx(node, info); SnitchContext context = getSnitchCtx(node, info, cc);
info.nodeVsContext.put(node, context); info.nodeVsContext.put(node, context);
try { try {
info.snitch.getTags(node, info.myTags, context); info.snitch.getTags(node, info.myTags, context);
@ -418,7 +418,7 @@ public class ReplicaAssigner {
if (context.exception != null) { if (context.exception != null) {
failedNodes.put(node, context); failedNodes.put(node, context);
participatingLiveNodes.remove(node); participatingLiveNodes.remove(node);
log.warn("Not all tags were obtained from node " + node); log.warn("Not all tags were obtained from node " + node, context.exception);
context.exception = new SolrException(SolrException.ErrorCode.SERVER_ERROR, context.exception = new SolrException(SolrException.ErrorCode.SERVER_ERROR,
"Not all tags were obtained from node " + node); "Not all tags were obtained from node " + node);
} else { } else {
@ -443,8 +443,9 @@ public class ReplicaAssigner {
} }
private Map<String, Object> snitchSession = new HashMap<>(); private Map<String, Object> snitchSession = new HashMap<>();
protected SnitchContext getSnitchCtx( String node, SnitchInfoImpl info) {
return new SnitchContext(info, node, snitchSession); protected SnitchContext getSnitchCtx(String node, SnitchInfoImpl info, CoreContainer cc) {
return new ServerSnitchContext(info, node, snitchSession, cc);
} }
public static void verifySnitchConf(CoreContainer cc, List snitchConf) { public static void verifySnitchConf(CoreContainer cc, List snitchConf) {

2
solr/core/src/java/org/apache/solr/cloud/rule/Rule.java
View File

@ -25,7 +25,7 @@ import org.apache.solr.common.SolrException;
import org.apache.solr.common.util.StrUtils; import org.apache.solr.common.util.StrUtils;
import org.apache.solr.common.util.Utils; import org.apache.solr.common.util.Utils;
import static org.apache.solr.cloud.rule.ImplicitSnitch.CORES; import static org.apache.solr.common.cloud.rule.ImplicitSnitch.CORES;
import static org.apache.solr.cloud.rule.Rule.MatchStatus.CANNOT_ASSIGN_FAIL; import static org.apache.solr.cloud.rule.Rule.MatchStatus.CANNOT_ASSIGN_FAIL;
import static org.apache.solr.cloud.rule.Rule.MatchStatus.NODE_CAN_BE_ASSIGNED; import static org.apache.solr.cloud.rule.Rule.MatchStatus.NODE_CAN_BE_ASSIGNED;
import static org.apache.solr.cloud.rule.Rule.MatchStatus.NOT_APPLICABLE; import static org.apache.solr.cloud.rule.Rule.MatchStatus.NOT_APPLICABLE;

90
solr/core/src/java/org/apache/solr/cloud/rule/SnitchContext.java → solr/core/src/java/org/apache/solr/cloud/rule/ServerSnitchContext.java
View File

@ -14,13 +14,12 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.solr.cloud.rule; package org.apache.solr.cloud.rule;
import java.io.IOException; import java.io.IOException;
import java.lang.invoke.MethodHandles; import java.lang.invoke.MethodHandles;
import java.util.HashMap;
import java.util.Map; import java.util.Map;
import java.util.Set;
import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.SolrServerException;
@ -28,6 +27,8 @@ import org.apache.solr.client.solrj.impl.BinaryResponseParser;
import org.apache.solr.client.solrj.impl.HttpSolrClient; import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.request.GenericSolrRequest; import org.apache.solr.client.solrj.request.GenericSolrRequest;
import org.apache.solr.client.solrj.response.SimpleSolrResponse; import org.apache.solr.client.solrj.response.SimpleSolrResponse;
import org.apache.solr.common.cloud.rule.RemoteCallback;
import org.apache.solr.common.cloud.rule.SnitchContext;
import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams; import org.apache.solr.common.params.SolrParams;
@ -42,47 +43,22 @@ import org.slf4j.LoggerFactory;
import static org.apache.solr.common.params.CoreAdminParams.ACTION; import static org.apache.solr.common.params.CoreAdminParams.ACTION;
import static org.apache.solr.common.params.CoreAdminParams.CoreAdminAction.INVOKE; import static org.apache.solr.common.params.CoreAdminParams.CoreAdminAction.INVOKE;
/** public class ServerSnitchContext extends SnitchContext {
* This is the context provided to the snitches to interact with the system. This is a per-node-per-snitch
* instance.
*/
public class SnitchContext implements RemoteCallback {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private final Map<String, Object> tags = new HashMap<>();
private String node;
private Map<String, Object> session;
final SnitchInfo snitchInfo;
Exception exception;
final CoreContainer coreContainer;
SnitchContext(SnitchInfo perSnitch, String node, Map<String, Object> session) { public ServerSnitchContext(SnitchInfo perSnitch,
this.snitchInfo = perSnitch; String node, Map<String, Object> session,
this.node = node; CoreContainer coreContainer) {
this.session = session; super(perSnitch, node, session);
this.coreContainer = coreContainer;
} }
public SnitchInfo getSnitchInfo() {
return snitchInfo;
}
public Map<String, Object> getTags() {
return tags;
}
public void store(String s, Object val) {
if (session != null) session.put(s, val);
}
public Object retrieve(String s) {
return session != null ? session.get(s) : null;
}
public Map getZkJson(String path) { public Map getZkJson(String path) {
if (snitchInfo.getCoreContainer().isZooKeeperAware()) { if (coreContainer.isZooKeeperAware()) {
try { try {
byte[] data = snitchInfo.getCoreContainer().getZkController().getZkClient().getData(path, null, new Stat(), true); byte[] data = coreContainer.getZkController().getZkClient().getData(path, null, new Stat(), true);
if (data == null) return null; if (data == null) return null;
return (Map) Utils.fromJSON(data); return (Map) Utils.fromJSON(data);
} catch (Exception e) { } catch (Exception e) {
@ -96,34 +72,20 @@ public class SnitchContext implements RemoteCallback {
} }
public String getNode() {
return node;
}
/**
* make a call to solrnode/admin/cores with the given params and give a callback. This is designed to be
* asynchronous because the system would want to batch the calls made to any given node
*
* @param node The node for which this call is made
* @param params The params to be passed to the Snitch counterpart
* @param klas The name of the class to be invoked in the remote node
* @param callback The callback to be called when the response is obtained from remote node.
* If this is passed as null the entire response map will be added as tags
*/
public void invokeRemote(String node, ModifiableSolrParams params, String klas, RemoteCallback callback) { public void invokeRemote(String node, ModifiableSolrParams params, String klas, RemoteCallback callback) {
if (callback == null) callback = this; if (callback == null) callback = this;
String url = snitchInfo.getCoreContainer().getZkController().getZkStateReader().getBaseUrlForNodeName(node); String url = coreContainer.getZkController().getZkStateReader().getBaseUrlForNodeName(node);
params.add("class", klas); params.add("class", klas);
params.add(ACTION, INVOKE.toString()); params.add(ACTION, INVOKE.toString());
//todo batch all requests to the same server //todo batch all requests to the same server
try { try {
SimpleSolrResponse rsp = invoke(snitchInfo.getCoreContainer().getUpdateShardHandler(), url, CommonParams.CORES_HANDLER_PATH, params); SimpleSolrResponse rsp = invoke(coreContainer.getUpdateShardHandler(), url, CommonParams.CORES_HANDLER_PATH, params);
Map<String, Object> returnedVal = (Map<String, Object>) rsp.getResponse().get(klas); Map<String, Object> returnedVal = (Map<String, Object>) rsp.getResponse().get(klas);
if(exception == null){ if(exception == null){
// log this // log this
} else { } else {
callback.remoteCallback(SnitchContext.this,returnedVal); callback.remoteCallback(ServerSnitchContext.this,returnedVal);
} }
callback.remoteCallback(this, returnedVal); callback.remoteCallback(this, returnedVal);
} catch (Exception e) { } catch (Exception e) {
@ -132,7 +94,6 @@ public class SnitchContext implements RemoteCallback {
} }
} }
public SimpleSolrResponse invoke(UpdateShardHandler shardHandler, final String url, String path, SolrParams params) public SimpleSolrResponse invoke(UpdateShardHandler shardHandler, final String url, String path, SolrParams params)
throws IOException, SolrServerException { throws IOException, SolrServerException {
GenericSolrRequest request = new GenericSolrRequest(SolrRequest.METHOD.GET, path, params); GenericSolrRequest request = new GenericSolrRequest(SolrRequest.METHOD.GET, path, params);
@ -144,25 +105,4 @@ public class SnitchContext implements RemoteCallback {
} }
} }
@Override
public void remoteCallback(SnitchContext ctx, Map<String, Object> returnedVal) {
tags.putAll(returnedVal);
}
public String getErrMsg() {
return exception == null ? null : exception.getMessage();
}
public static abstract class SnitchInfo {
private final Map<String, Object> conf;
SnitchInfo(Map<String, Object> conf) {
this.conf = conf;
}
public abstract Set<String> getTagNames();
public abstract CoreContainer getCoreContainer();
}
} }

2
solr/core/src/java/org/apache/solr/cloud/rule/Snitch.java
View File

@ -27,7 +27,7 @@ public abstract class Snitch {
static Set<Class> WELL_KNOWN_SNITCHES = ImmutableSet.of(ImplicitSnitch.class); static Set<Class> WELL_KNOWN_SNITCHES = ImmutableSet.of(ImplicitSnitch.class);
public abstract void getTags(String solrNode, Set<String> requestedTags, SnitchContext ctx); public abstract void getTags(String solrNode, Set<String> requestedTags, org.apache.solr.common.cloud.rule.SnitchContext ctx);
public abstract boolean isKnownTag(String tag); public abstract boolean isKnownTag(String tag);

16
solr/core/src/test/org/apache/solr/cloud/rule/ImplicitSnitchTest.java
View File

@ -21,6 +21,8 @@ import java.util.HashMap;
import java.util.Map; import java.util.Map;
import com.google.common.collect.Sets; import com.google.common.collect.Sets;
import org.apache.solr.common.cloud.rule.ImplicitSnitch;
import org.apache.solr.common.cloud.rule.SnitchContext;
import org.junit.Before; import org.junit.Before;
import org.junit.Test; import org.junit.Test;
import org.mockito.Mockito; import org.mockito.Mockito;
@ -45,7 +47,7 @@ public class ImplicitSnitchTest {
@Before @Before
public void beforeImplicitSnitchTest() { public void beforeImplicitSnitchTest() {
snitch = new ImplicitSnitch(); snitch = new ImplicitSnitch();
context = new SnitchContext(null, null, new HashMap<>()); context = new ServerSnitchContext(null, null, new HashMap<>(),null);
} }
@ -82,7 +84,7 @@ public class ImplicitSnitchTest {
public void testGetTags_withIPv4RequestedTags_ip2_and_ip4_returns_two_tags() throws Exception { public void testGetTags_withIPv4RequestedTags_ip2_and_ip4_returns_two_tags() throws Exception {
String node = "192.168.1.2:8983_solr"; String node = "192.168.1.2:8983_solr";
SnitchContext context = new SnitchContext(null, node, new HashMap<>()); SnitchContext context = new ServerSnitchContext(null, node, new HashMap<>(),null);
snitch.getTags(node, Sets.newHashSet(IP_2, IP_4), context); snitch.getTags(node, Sets.newHashSet(IP_2, IP_4), context);
Map<String, Object> tags = context.getTags(); Map<String, Object> tags = context.getTags();
@ -95,7 +97,7 @@ public class ImplicitSnitchTest {
public void testGetTags_with_wrong_ipv4_format_ip_returns_nothing() throws Exception { public void testGetTags_with_wrong_ipv4_format_ip_returns_nothing() throws Exception {
String node = "192.168.1.2.1:8983_solr"; String node = "192.168.1.2.1:8983_solr";
SnitchContext context = new SnitchContext(null, node, new HashMap<>()); SnitchContext context = new ServerSnitchContext(null, node, new HashMap<>(),null);
snitch.getTags(node, Sets.newHashSet(IP_1), context); snitch.getTags(node, Sets.newHashSet(IP_1), context);
Map<String, Object> tags = context.getTags(); Map<String, Object> tags = context.getTags();
@ -107,7 +109,7 @@ public class ImplicitSnitchTest {
public void testGetTags_with_correct_ipv6_format_ip_returns_nothing() throws Exception { public void testGetTags_with_correct_ipv6_format_ip_returns_nothing() throws Exception {
String node = "[0:0:0:0:0:0:0:1]:8983_solr"; String node = "[0:0:0:0:0:0:0:1]:8983_solr";
SnitchContext context = new SnitchContext(null, node, new HashMap<>()); SnitchContext context = new ServerSnitchContext(null, node, new HashMap<>(),null);
snitch.getTags(node, Sets.newHashSet(IP_1), context); snitch.getTags(node, Sets.newHashSet(IP_1), context);
Map<String, Object> tags = context.getTags(); Map<String, Object> tags = context.getTags();
@ -130,7 +132,7 @@ public class ImplicitSnitchTest {
public void testGetTags_withAllHostNameRequestedTags_returns_all_Tags() throws Exception { public void testGetTags_withAllHostNameRequestedTags_returns_all_Tags() throws Exception {
String node = "serv01.dc01.london.uk.apache.org:8983_solr"; String node = "serv01.dc01.london.uk.apache.org:8983_solr";
SnitchContext context = new SnitchContext(null, node, new HashMap<>()); SnitchContext context = new ServerSnitchContext(null, node, new HashMap<>(),null);
//We need mocking here otherwise, we would need proper DNS entry for this test to pass //We need mocking here otherwise, we would need proper DNS entry for this test to pass
ImplicitSnitch mockedSnitch = Mockito.spy(snitch); ImplicitSnitch mockedSnitch = Mockito.spy(snitch);
when(mockedSnitch.getHostIp(anyString())).thenReturn("10.11.12.13"); when(mockedSnitch.getHostIp(anyString())).thenReturn("10.11.12.13");
@ -149,7 +151,7 @@ public class ImplicitSnitchTest {
public void testGetTags_withHostNameRequestedTag_ip3_returns_1_tag() throws Exception { public void testGetTags_withHostNameRequestedTag_ip3_returns_1_tag() throws Exception {
String node = "serv01.dc01.london.uk.apache.org:8983_solr"; String node = "serv01.dc01.london.uk.apache.org:8983_solr";
SnitchContext context = new SnitchContext(null, node, new HashMap<>()); SnitchContext context = new ServerSnitchContext(null, node, new HashMap<>(),null);
//We need mocking here otherwise, we would need proper DNS entry for this test to pass //We need mocking here otherwise, we would need proper DNS entry for this test to pass
ImplicitSnitch mockedSnitch = Mockito.spy(snitch); ImplicitSnitch mockedSnitch = Mockito.spy(snitch);
when(mockedSnitch.getHostIp(anyString())).thenReturn("10.11.12.13"); when(mockedSnitch.getHostIp(anyString())).thenReturn("10.11.12.13");
@ -164,7 +166,7 @@ public class ImplicitSnitchTest {
public void testGetTags_withHostNameRequestedTag_ip99999_returns_nothing() throws Exception { public void testGetTags_withHostNameRequestedTag_ip99999_returns_nothing() throws Exception {
String node = "serv01.dc01.london.uk.apache.org:8983_solr"; String node = "serv01.dc01.london.uk.apache.org:8983_solr";
SnitchContext context = new SnitchContext(null, node, new HashMap<>()); SnitchContext context = new ServerSnitchContext(null, node, new HashMap<>(),null);
//We need mocking here otherwise, we would need proper DNS entry for this test to pass //We need mocking here otherwise, we would need proper DNS entry for this test to pass
ImplicitSnitch mockedSnitch = Mockito.spy(snitch); ImplicitSnitch mockedSnitch = Mockito.spy(snitch);
when(mockedSnitch.getHostIp(anyString())).thenReturn("10.11.12.13"); when(mockedSnitch.getHostIp(anyString())).thenReturn("10.11.12.13");

9
solr/core/src/test/org/apache/solr/cloud/rule/RuleEngineTest.java
View File

@ -30,7 +30,10 @@ import com.google.common.collect.ImmutableList;
import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.cloud.rule.ReplicaAssigner.Position; import org.apache.solr.cloud.rule.ReplicaAssigner.Position;
import org.apache.solr.common.cloud.ZkStateReader; import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.cloud.rule.Snitch;
import org.apache.solr.common.cloud.rule.SnitchContext;
import org.apache.solr.common.util.Utils; import org.apache.solr.common.util.Utils;
import org.apache.solr.core.CoreContainer;
import org.junit.Test; import org.junit.Test;
import static java.util.Collections.singletonList; import static java.util.Collections.singletonList;
@ -91,8 +94,8 @@ public class RuleEngineTest extends SolrTestCaseJ4{
new HashMap(), new ArrayList<>(MockSnitch.nodeVsTags.keySet()), null, null) { new HashMap(), new ArrayList<>(MockSnitch.nodeVsTags.keySet()), null, null) {
@Override @Override
protected SnitchContext getSnitchCtx(String node, SnitchInfoImpl info) { protected SnitchContext getSnitchCtx(String node, SnitchInfoImpl info, CoreContainer cc) {
return new SnitchContext(info, node, snitchSession){ return new ServerSnitchContext(info, node, snitchSession,cc){
@Override @Override
public Map getZkJson(String path) { public Map getZkJson(String path) {
if(ZkStateReader.ROLES.equals(path)){ if(ZkStateReader.ROLES.equals(path)){
@ -100,8 +103,10 @@ public class RuleEngineTest extends SolrTestCaseJ4{
} }
return null; return null;
} }
}; };
} }
}; };
mapping = replicaAssigner.getNodeMappings(); mapping = replicaAssigner.getNodeMappings();
assertNotNull(mapping); assertNotNull(mapping);

163
solr/solrj/src/java/org/apache/solr/common/cloud/rule/ImplicitSnitch.java Normal file
View File

@ -0,0 +1,163 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.common.cloud.rule;
import java.lang.invoke.MethodHandles;
import java.net.InetAddress;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
//This is the client-side component of the snitch
public class ImplicitSnitch extends Snitch {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
public static final Pattern hostAndPortPattern = Pattern.compile("(?:https?://)?([^:]+):(\\d+)");
//well known tags
public static final String NODE = "node";
public static final String PORT = "port";
public static final String HOST = "host";
public static final String CORES = "cores";
public static final String DISK = "freedisk";
public static final String ROLE = "role";
public static final String SYSPROP = "sysprop.";
public static final List<String> IP_SNITCHES = ImmutableList.of("ip_1", "ip_2", "ip_3", "ip_4");
public static final Set<String> tags = ImmutableSet.<String>builder().add(NODE, PORT, HOST, CORES, DISK, ROLE).addAll(IP_SNITCHES).build();
@Override
public void getTags(String solrNode, Set<String> requestedTags, SnitchContext ctx) {
if (requestedTags.contains(NODE)) ctx.getTags().put(NODE, solrNode);
if (requestedTags.contains(HOST)) {
Matcher hostAndPortMatcher = hostAndPortPattern.matcher(solrNode);
if (hostAndPortMatcher.find()) ctx.getTags().put(HOST, hostAndPortMatcher.group(1));
}
if (requestedTags.contains(PORT)) {
Matcher hostAndPortMatcher = hostAndPortPattern.matcher(solrNode);
if (hostAndPortMatcher.find()) ctx.getTags().put(PORT, hostAndPortMatcher.group(2));
}
if (requestedTags.contains(ROLE)) fillRole(solrNode, ctx);
addIpTags(solrNode, requestedTags, ctx);
ModifiableSolrParams params = new ModifiableSolrParams();
if (requestedTags.contains(CORES)) params.add(CORES, "1");
if (requestedTags.contains(DISK)) params.add(DISK, "1");
for (String tag : requestedTags) {
if (tag.startsWith(SYSPROP)) params.add(SYSPROP, tag.substring(SYSPROP.length()));
}
if (params.size() > 0) ctx.invokeRemote(solrNode, params, "org.apache.solr.cloud.rule.ImplicitSnitch", null);
}
private void fillRole(String solrNode, SnitchContext ctx) {
Map roles = (Map) ctx.retrieve(ZkStateReader.ROLES); // we don't want to hit the ZK for each node
// so cache and reuse
if(roles == null) roles = ctx.getZkJson(ZkStateReader.ROLES);
ctx.store(ZkStateReader.ROLES, roles == null ? Collections.emptyMap() : roles);
if (roles != null) {
for (Object o : roles.entrySet()) {
Map.Entry e = (Map.Entry) o;
if (e.getValue() instanceof List) {
if(((List) e.getValue()).contains(solrNode)) {
ctx.getTags().put(ROLE, e.getKey());
break;
}
}
}
}
}
private static final String HOST_FRAG_SEPARATOR_REGEX = "\\.";
@Override
public boolean isKnownTag(String tag) {
return tags.contains(tag) ||
tag.startsWith(SYSPROP);
}
private void addIpTags(String solrNode, Set<String> requestedTags, SnitchContext context) {
List<String> requestedHostTags = new ArrayList<>();
for (String tag : requestedTags) {
if (IP_SNITCHES.contains(tag)) {
requestedHostTags.add(tag);
}
}
if (requestedHostTags.isEmpty()) {
return;
}
String[] ipFragments = getIpFragments(solrNode);
if (ipFragments == null) {
return;
}
int ipSnitchCount = IP_SNITCHES.size();
for (int i = 0; i < ipSnitchCount; i++) {
String currentTagValue = ipFragments[i];
String currentTagKey = IP_SNITCHES.get(ipSnitchCount - i - 1);
if (requestedHostTags.contains(currentTagKey)) {
context.getTags().put(currentTagKey, currentTagValue);
}
}
}
private String[] getIpFragments(String solrNode) {
Matcher hostAndPortMatcher = hostAndPortPattern.matcher(solrNode);
if (hostAndPortMatcher.find()) {
String host = hostAndPortMatcher.group(1);
if (host != null) {
String ip = getHostIp(host);
if (ip != null) {
return ip.split(HOST_FRAG_SEPARATOR_REGEX); //IPv6 support will be provided by SOLR-8523
}
}
}
log.warn("Failed to match host IP address from node URL [{}] using regex [{}]", solrNode, hostAndPortPattern.pattern());
return null;
}
public String getHostIp(String host) {
try {
InetAddress address = InetAddress.getByName(host);
return address.getHostAddress();
} catch (Exception e) {
log.warn("Failed to get IP address from host [{}], with exception [{}] ", host, e);
return null;
}
}
}

2
solr/core/src/java/org/apache/solr/cloud/rule/RemoteCallback.java → solr/solrj/src/java/org/apache/solr/common/cloud/rule/RemoteCallback.java
View File

@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.solr.cloud.rule; package org.apache.solr.common.cloud.rule;
import java.util.Map; import java.util.Map;

34
solr/solrj/src/java/org/apache/solr/common/cloud/rule/Snitch.java Normal file
View File

@ -0,0 +1,34 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.common.cloud.rule;
import java.util.Set;
import com.google.common.collect.ImmutableSet;
/**
*
*/
public abstract class Snitch {
public static final Set<Class> WELL_KNOWN_SNITCHES = ImmutableSet.of(ImplicitSnitch.class);
public abstract void getTags(String solrNode, Set<String> requestedTags, SnitchContext ctx);
public abstract boolean isKnownTag(String tag);
}

103
solr/solrj/src/java/org/apache/solr/common/cloud/rule/SnitchContext.java Normal file
View File

@ -0,0 +1,103 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.common.cloud.rule;
import java.lang.invoke.MethodHandles;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* This is the context provided to the snitches to interact with the system. This is a per-node-per-snitch
* instance.
*/
public abstract class SnitchContext implements RemoteCallback {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private final Map<String, Object> tags = new HashMap<>();
private String node;
private Map<String, Object> session;
public final SnitchInfo snitchInfo;
public Exception exception;
public SnitchContext(SnitchInfo perSnitch, String node, Map<String, Object> session) {
this.snitchInfo = perSnitch;
this.node = node;
this.session = session;
}
public SnitchInfo getSnitchInfo() {
return snitchInfo;
}
public Map<String, Object> getTags() {
return tags;
}
public void store(String s, Object val) {
if (session != null) session.put(s, val);
}
public Object retrieve(String s) {
return session != null ? session.get(s) : null;
}
public abstract Map getZkJson(String path) ;
public String getNode() {
return node;
}
/**
* make a call to solrnode/admin/cores with the given params and give a callback. This is designed to be
* asynchronous because the system would want to batch the calls made to any given node
*
* @param node The node for which this call is made
* @param params The params to be passed to the Snitch counterpart
* @param klas The name of the class to be invoked in the remote node
* @param callback The callback to be called when the response is obtained from remote node.
* If this is passed as null the entire response map will be added as tags
*/
public abstract void invokeRemote(String node, ModifiableSolrParams params, String klas, RemoteCallback callback) ;
@Override
public void remoteCallback(SnitchContext ctx, Map<String, Object> returnedVal) {
tags.putAll(returnedVal);
}
public String getErrMsg() {
return exception == null ? null : exception.getMessage();
}
public static abstract class SnitchInfo {
private final Map<String, Object> conf;
protected SnitchInfo(Map<String, Object> conf) {
this.conf = conf;
}
public abstract Set<String> getTagNames();
}
}

23
solr/solrj/src/java/org/apache/solr/common/cloud/rule/package-info.java Normal file
View File

@ -0,0 +1,23 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* Classes for managing Replica placement strategy when operating in <a href="http://wiki.apache.org/solr/SolrCloud">SolrCloud</a> mode.
*/
package org.apache.solr.common.cloud.rule;