mirror of https://github.com/apache/lucene.git
SOLR-8522: Make it possible to use ip fragments in replica placement rules , such as ip_1, ip_2 etc
This commit is contained in:
Noble Paul
parent
e069d31086
commit
cf96432630
|
|
@ -153,6 +153,8 @@ New Features
|
|||
a connection to zookeeper has been lost and there is a possibility of stale data on the node the request is coming
|
||||
from. (Keith Laban, Dennis Gove)
|
||||
|
||||
* SOLR-8522: Make it possible to use ip fragments in replica placement rules , such as ip_1, ip_2 etc (Arcadius Ahouansou, noble)
|
||||
|
||||
Bug Fixes
|
||||
----------------------
|
||||
* SOLR-8386: Add field option in the new admin UI schema page loads up even when no schemaFactory has been
|
||||
|
|
|
|||
|
|
@ -54,6 +54,8 @@
|
|||
<dependency org="cglib" name="cglib-nodep" rev="${/cglib/cglib-nodep}" conf="test"/>
|
||||
<dependency org="org.objenesis" name="objenesis" rev="${/org.objenesis/objenesis}" conf="test"/>
|
||||
|
||||
<dependency org="org.mockito" name="mockito-core" rev="${/org.mockito/mockito-core}" conf="test"/>
|
||||
|
||||
<dependency org="com.fasterxml.jackson.core" name="jackson-core" rev="${/com.fasterxml.jackson.core/jackson-core}" conf="compile"/>
|
||||
<dependency org="com.fasterxml.jackson.core" name="jackson-databind" rev="${/com.fasterxml.jackson.core/jackson-databind}" conf="test"/>
|
||||
<dependency org="com.fasterxml.jackson.core" name="jackson-annotations" rev="${/com.fasterxml.jackson.core/jackson-annotations}" conf="test"/>
|
||||
|
|
|
|||
|
|
@ -17,21 +17,29 @@
|
|||
package org.apache.solr.cloud.rule;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.lang.invoke.MethodHandles;
|
||||
import java.net.InetAddress;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import org.apache.solr.common.params.ModifiableSolrParams;
|
||||
import org.apache.solr.core.CoreContainer;
|
||||
import org.apache.solr.handler.admin.CoreAdminHandler;
|
||||
import org.apache.solr.request.SolrQueryRequest;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
public class ImplicitSnitch extends Snitch implements CoreAdminHandler.Invocable {
|
||||
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
|
||||
|
||||
public static final Pattern hostAndPortPattern = Pattern.compile("(?:https?://)?([^:]+):(\\d+)");
|
||||
|
||||
|
|
@ -42,8 +50,10 @@ public class ImplicitSnitch extends Snitch implements CoreAdminHandler.Invocable
|
|||
public static final String CORES = "cores";
|
||||
public static final String DISK = "freedisk";
|
||||
public static final String SYSPROP = "sysprop.";
|
||||
public static final List<String> IP_SNITCHES = ImmutableList.of("ip_1", "ip_2", "ip_3", "ip_4");
|
||||
|
||||
public static final Set<String> tags = ImmutableSet.<String>builder().add(NODE, PORT, HOST, CORES, DISK).addAll(IP_SNITCHES).build();
|
||||
|
||||
public static final Set<String> tags = ImmutableSet.of(NODE, PORT, HOST, CORES, DISK);
|
||||
|
||||
|
||||
@Override
|
||||
|
|
@ -57,6 +67,9 @@ public class ImplicitSnitch extends Snitch implements CoreAdminHandler.Invocable
|
|||
Matcher hostAndPortMatcher = hostAndPortPattern.matcher(solrNode);
|
||||
if (hostAndPortMatcher.find()) ctx.getTags().put(PORT, hostAndPortMatcher.group(2));
|
||||
}
|
||||
|
||||
addIpTags(solrNode, requestedTags, ctx);
|
||||
|
||||
ModifiableSolrParams params = new ModifiableSolrParams();
|
||||
if (requestedTags.contains(CORES)) params.add(CORES, "1");
|
||||
if (requestedTags.contains(DISK)) params.add(DISK, "1");
|
||||
|
|
@ -71,7 +84,7 @@ public class ImplicitSnitch extends Snitch implements CoreAdminHandler.Invocable
|
|||
long spaceInGB = space / 1024 / 1024 / 1024;
|
||||
return spaceInGB;
|
||||
}
|
||||
|
||||
|
||||
public Map<String, Object> invoke(SolrQueryRequest req) {
|
||||
Map<String, Object> result = new HashMap<>();
|
||||
if (req.getParams().getInt(CORES, -1) == 1) {
|
||||
|
|
@ -88,16 +101,75 @@ public class ImplicitSnitch extends Snitch implements CoreAdminHandler.Invocable
|
|||
}
|
||||
String[] sysProps = req.getParams().getParams(SYSPROP);
|
||||
if (sysProps != null && sysProps.length > 0) {
|
||||
for (String prop : sysProps) result.put(SYSPROP+prop, System.getProperty(prop));
|
||||
for (String prop : sysProps) result.put(SYSPROP + prop, System.getProperty(prop));
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private static final String HOST_FRAG_SEPARATOR_REGEX = "\\.";
|
||||
|
||||
@Override
|
||||
public boolean isKnownTag(String tag) {
|
||||
return tags.contains(tag) ||
|
||||
tag.startsWith(SYSPROP);//a system property
|
||||
tag.startsWith(SYSPROP);
|
||||
}
|
||||
|
||||
private void addIpTags(String solrNode, Set<String> requestedTags, SnitchContext context) {
|
||||
|
||||
List<String> requestedHostTags = new ArrayList<>();
|
||||
for (String tag : requestedTags) {
|
||||
if (IP_SNITCHES.contains(tag)) {
|
||||
requestedHostTags.add(tag);
|
||||
}
|
||||
}
|
||||
|
||||
if (requestedHostTags.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
|
||||
String[] ipFragments = getIpFragments(solrNode);
|
||||
|
||||
if (ipFragments == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int ipSnitchCount = IP_SNITCHES.size();
|
||||
for (int i = 0; i < ipSnitchCount; i++) {
|
||||
String currentTagValue = ipFragments[i];
|
||||
String currentTagKey = IP_SNITCHES.get(ipSnitchCount - i - 1);
|
||||
|
||||
if (requestedHostTags.contains(currentTagKey)) {
|
||||
context.getTags().put(currentTagKey, currentTagValue);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
private String[] getIpFragments(String solrNode) {
|
||||
Matcher hostAndPortMatcher = hostAndPortPattern.matcher(solrNode);
|
||||
if (hostAndPortMatcher.find()) {
|
||||
String host = hostAndPortMatcher.group(1);
|
||||
if (host != null) {
|
||||
String ip = getHostIp(host);
|
||||
if (ip != null) {
|
||||
return ip.split(HOST_FRAG_SEPARATOR_REGEX); //IPv6 support will be provided by SOLR-8523
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
log.warn("Failed to match host IP address from node URL [{}] using regex [{}]", solrNode, hostAndPortPattern.pattern());
|
||||
return null;
|
||||
}
|
||||
|
||||
protected String getHostIp(String host) {
|
||||
try {
|
||||
InetAddress address = InetAddress.getByName(host);
|
||||
return address.getHostAddress();
|
||||
} catch (Exception e) {
|
||||
log.warn("Failed to get IP address from host [{}], with exception [{}] ", host, e);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,186 @@
|
|||
package org.apache.solr.cloud.rule;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
import com.google.common.collect.Sets;
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
import org.mockito.Mockito;
|
||||
|
||||
import static org.hamcrest.core.Is.is;
|
||||
import static org.junit.Assert.assertFalse;
|
||||
import static org.junit.Assert.assertThat;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
import static org.mockito.Matchers.anyString;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
public class ImplicitSnitchTest {
|
||||
|
||||
private ImplicitSnitch snitch;
|
||||
private SnitchContext context;
|
||||
|
||||
private static final String IP_1 = "ip_1";
|
||||
private static final String IP_2 = "ip_2";
|
||||
private static final String IP_3 = "ip_3";
|
||||
private static final String IP_4 = "ip_4";
|
||||
|
||||
@Before
|
||||
public void beforeImplicitSnitchTest() {
|
||||
snitch = new ImplicitSnitch();
|
||||
context = new SnitchContext(null, null);
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testGetTags_withAllIPv4RequestedTags_with_omitted_zeros_returns_four_tags() throws Exception {
|
||||
String node = "5:8983_solr";
|
||||
|
||||
snitch.getTags(node, Sets.newHashSet(IP_1, IP_2, IP_3, IP_4), context);
|
||||
|
||||
Map<String, Object> tags = context.getTags();
|
||||
assertThat(tags.entrySet().size(), is(4));
|
||||
assertThat(tags.get(IP_1), is("5"));
|
||||
assertThat(tags.get(IP_2), is("0"));
|
||||
assertThat(tags.get(IP_3), is("0"));
|
||||
assertThat(tags.get(IP_4), is("0"));
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testGetTags_withAllIPv4RequestedTags_returns_four_tags() throws Exception {
|
||||
String node = "192.168.1.2:8983_solr";
|
||||
|
||||
snitch.getTags(node, Sets.newHashSet(IP_1, IP_2, IP_3, IP_4), context);
|
||||
|
||||
Map<String, Object> tags = context.getTags();
|
||||
assertThat(tags.entrySet().size(), is(4));
|
||||
assertThat(tags.get(IP_1), is("2"));
|
||||
assertThat(tags.get(IP_2), is("1"));
|
||||
assertThat(tags.get(IP_3), is("168"));
|
||||
assertThat(tags.get(IP_4), is("192"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetTags_withIPv4RequestedTags_ip2_and_ip4_returns_two_tags() throws Exception {
|
||||
String node = "192.168.1.2:8983_solr";
|
||||
|
||||
SnitchContext context = new SnitchContext(null, node);
|
||||
snitch.getTags(node, Sets.newHashSet(IP_2, IP_4), context);
|
||||
|
||||
Map<String, Object> tags = context.getTags();
|
||||
assertThat(tags.entrySet().size(), is(2));
|
||||
assertThat(tags.get(IP_2), is("1"));
|
||||
assertThat(tags.get(IP_4), is("192"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetTags_with_wrong_ipv4_format_ip_returns_nothing() throws Exception {
|
||||
String node = "192.168.1.2.1:8983_solr";
|
||||
|
||||
SnitchContext context = new SnitchContext(null, node);
|
||||
snitch.getTags(node, Sets.newHashSet(IP_1), context);
|
||||
|
||||
Map<String, Object> tags = context.getTags();
|
||||
assertThat(tags.entrySet().size(), is(0));
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testGetTags_with_correct_ipv6_format_ip_returns_nothing() throws Exception {
|
||||
String node = "[0:0:0:0:0:0:0:1]:8983_solr";
|
||||
|
||||
SnitchContext context = new SnitchContext(null, node);
|
||||
snitch.getTags(node, Sets.newHashSet(IP_1), context);
|
||||
|
||||
Map<String, Object> tags = context.getTags();
|
||||
assertThat(tags.entrySet().size(), is(0)); //This will fail when IPv6 is implemented
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testGetTags_withEmptyRequestedTag_returns_nothing() throws Exception {
|
||||
String node = "192.168.1.2:8983_solr";
|
||||
|
||||
snitch.getTags(node, Sets.newHashSet(), context);
|
||||
|
||||
Map<String, Object> tags = context.getTags();
|
||||
assertThat(tags.entrySet().size(), is(0));
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testGetTags_withAllHostNameRequestedTags_returns_all_Tags() throws Exception {
|
||||
String node = "serv01.dc01.london.uk.apache.org:8983_solr";
|
||||
|
||||
SnitchContext context = new SnitchContext(null, node);
|
||||
//We need mocking here otherwise, we would need proper DNS entry for this test to pass
|
||||
ImplicitSnitch mockedSnitch = Mockito.spy(snitch);
|
||||
when(mockedSnitch.getHostIp(anyString())).thenReturn("10.11.12.13");
|
||||
|
||||
mockedSnitch.getTags(node, Sets.newHashSet(IP_1, IP_2, IP_3, IP_4), context);
|
||||
|
||||
Map<String, Object> tags = context.getTags();
|
||||
assertThat(tags.entrySet().size(), is(4));
|
||||
assertThat(tags.get(IP_1), is("13"));
|
||||
assertThat(tags.get(IP_2), is("12"));
|
||||
assertThat(tags.get(IP_3), is("11"));
|
||||
assertThat(tags.get(IP_4), is("10"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetTags_withHostNameRequestedTag_ip3_returns_1_tag() throws Exception {
|
||||
String node = "serv01.dc01.london.uk.apache.org:8983_solr";
|
||||
|
||||
SnitchContext context = new SnitchContext(null, node);
|
||||
//We need mocking here otherwise, we would need proper DNS entry for this test to pass
|
||||
ImplicitSnitch mockedSnitch = Mockito.spy(snitch);
|
||||
when(mockedSnitch.getHostIp(anyString())).thenReturn("10.11.12.13");
|
||||
mockedSnitch.getTags(node, Sets.newHashSet(IP_3), context);
|
||||
|
||||
Map<String, Object> tags = context.getTags();
|
||||
assertThat(tags.entrySet().size(), is(1));
|
||||
assertThat(tags.get(IP_3), is("11"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetTags_withHostNameRequestedTag_ip99999_returns_nothing() throws Exception {
|
||||
String node = "serv01.dc01.london.uk.apache.org:8983_solr";
|
||||
|
||||
SnitchContext context = new SnitchContext(null, node);
|
||||
//We need mocking here otherwise, we would need proper DNS entry for this test to pass
|
||||
ImplicitSnitch mockedSnitch = Mockito.spy(snitch);
|
||||
when(mockedSnitch.getHostIp(anyString())).thenReturn("10.11.12.13");
|
||||
mockedSnitch.getTags(node, Sets.newHashSet("ip_99999"), context);
|
||||
|
||||
Map<String, Object> tags = context.getTags();
|
||||
assertThat(tags.entrySet().size(), is(0));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testIsKnownTag_ip1() throws Exception {
|
||||
assertFalse(snitch.isKnownTag("ip_0"));
|
||||
assertTrue(snitch.isKnownTag(IP_1));
|
||||
assertTrue(snitch.isKnownTag(IP_2));
|
||||
assertTrue(snitch.isKnownTag(IP_3));
|
||||
assertTrue(snitch.isKnownTag(IP_4));
|
||||
assertFalse(snitch.isKnownTag("ip_5"));
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -34,15 +34,20 @@ import org.apache.solr.common.cloud.ImplicitDocRouter;
|
|||
import org.apache.solr.common.cloud.ZkStateReader;
|
||||
import org.apache.solr.common.params.ModifiableSolrParams;
|
||||
import org.junit.Test;
|
||||
import org.junit.rules.ExpectedException;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import static org.apache.solr.client.solrj.SolrRequest.METHOD.POST;
|
||||
import static org.apache.solr.common.params.CommonParams.COLLECTIONS_HANDLER_PATH;
|
||||
import static org.junit.matchers.JUnitMatchers.containsString;
|
||||
|
||||
public class RulesTest extends AbstractFullDistribZkTestBase {
|
||||
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
|
||||
|
||||
@org.junit.Rule
|
||||
public ExpectedException expectedException = ExpectedException.none();
|
||||
|
||||
@Test
|
||||
@ShardsFixed(num = 5)
|
||||
public void doIntegrationTest() throws Exception {
|
||||
|
|
@ -127,6 +132,78 @@ public class RulesTest extends AbstractFullDistribZkTestBase {
|
|||
assertEquals ( "ImplicitSnitch", ((Map)list.get(0)).get("class"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testHostFragmentRule() throws Exception {
|
||||
String rulesColl = "ipRuleColl";
|
||||
String baseUrl = getBaseUrl((HttpSolrClient) clients.get(0));
|
||||
String ip_1 = "-1";
|
||||
String ip_2 = "-1";
|
||||
Matcher hostAndPortMatcher = Pattern.compile("(?:https?://)?([^:]+):(\\d+)").matcher(baseUrl);
|
||||
if (hostAndPortMatcher.find()) {
|
||||
String[] ipFragments = hostAndPortMatcher.group(1).split("\\.");
|
||||
ip_1 = ipFragments[ipFragments.length - 1];
|
||||
ip_2 = ipFragments[ipFragments.length - 2];
|
||||
}
|
||||
|
||||
try (SolrClient client = createNewSolrClient("", baseUrl)) {
|
||||
CollectionAdminResponse rsp;
|
||||
CollectionAdminRequest.Create create = new CollectionAdminRequest.Create();
|
||||
create.setCollectionName(rulesColl);
|
||||
create.setShards("shard1");
|
||||
create.setRouterName(ImplicitDocRouter.NAME);
|
||||
create.setReplicationFactor(2);
|
||||
create.setRule("ip_2:" + ip_2, "ip_1:" + ip_1);
|
||||
create.setSnitch("class:ImplicitSnitch");
|
||||
rsp = create.process(client);
|
||||
assertEquals(0, rsp.getStatus());
|
||||
assertTrue(rsp.isSuccess());
|
||||
|
||||
}
|
||||
|
||||
DocCollection rulesCollection = cloudClient.getZkStateReader().getClusterState().getCollection(rulesColl);
|
||||
List<Map> list = (List<Map>) rulesCollection.get("rule");
|
||||
assertEquals(2, list.size());
|
||||
assertEquals(ip_2, list.get(0).get("ip_2"));
|
||||
assertEquals(ip_1, list.get(1).get("ip_1"));
|
||||
|
||||
list = (List) rulesCollection.get("snitch");
|
||||
assertEquals(1, list.size());
|
||||
assertEquals("ImplicitSnitch", list.get(0).get("class"));
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testHostFragmentRuleThrowsExceptionWhenIpDoesNotMatch() throws Exception {
|
||||
String rulesColl = "ipRuleColl";
|
||||
String baseUrl = getBaseUrl((HttpSolrClient) clients.get(0));
|
||||
String ip_1 = "-1";
|
||||
String ip_2 = "-1";
|
||||
Matcher hostAndPortMatcher = Pattern.compile("(?:https?://)?([^:]+):(\\d+)").matcher(baseUrl);
|
||||
if (hostAndPortMatcher.find()) {
|
||||
String[] ipFragments = hostAndPortMatcher.group(1).split("\\.");
|
||||
ip_1 = ipFragments[ipFragments.length - 1];
|
||||
ip_2 = ipFragments[ipFragments.length - 2];
|
||||
}
|
||||
|
||||
try (SolrClient client = createNewSolrClient("", baseUrl)) {
|
||||
CollectionAdminRequest.Create create = new CollectionAdminRequest.Create();
|
||||
create.setCollectionName(rulesColl);
|
||||
create.setShards("shard1");
|
||||
create.setRouterName(ImplicitDocRouter.NAME);
|
||||
create.setReplicationFactor(2);
|
||||
|
||||
create.setRule("ip_2:" + ip_2, "ip_1:" + ip_1 + "9999");
|
||||
create.setSnitch("class:ImplicitSnitch");
|
||||
|
||||
expectedException.expect(HttpSolrClient.RemoteSolrException.class);
|
||||
expectedException.expectMessage(containsString("ip_1"));
|
||||
|
||||
create.process(client);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testModifyColl() throws Exception {
|
||||
final long minGB1 = (random().nextBoolean() ? 1 : 0);
|
||||
|
|
|
|||
Loading…
Reference in New Issue