SOLR-8522: Make it possible to use ip fragments in replica placement rules , such as ip_1, ip_2 etc

solr/CHANGES.txt

@@ -153,6 +153,8 @@ New Features
   a connection to zookeeper has been lost and there is a possibility of stale data on the node the request is coming
   from. (Keith Laban, Dennis Gove)
 
+* SOLR-8522: Make it possible to use ip fragments in replica placement rules , such as ip_1, ip_2 etc (Arcadius Ahouansou, noble)
+
 Bug Fixes
 ----------------------
 * SOLR-8386: Add field option in the new admin UI schema page loads up even when no schemaFactory has been

solr/core/ivy.xml

@@ -54,6 +54,8 @@
     <dependency org="cglib" name="cglib-nodep" rev="${/cglib/cglib-nodep}" conf="test"/>
     <dependency org="org.objenesis" name="objenesis" rev="${/org.objenesis/objenesis}" conf="test"/>
 
+    <dependency org="org.mockito" name="mockito-core" rev="${/org.mockito/mockito-core}" conf="test"/>
+
     <dependency org="com.fasterxml.jackson.core" name="jackson-core" rev="${/com.fasterxml.jackson.core/jackson-core}" conf="compile"/>
     <dependency org="com.fasterxml.jackson.core" name="jackson-databind" rev="${/com.fasterxml.jackson.core/jackson-databind}" conf="test"/>
     <dependency org="com.fasterxml.jackson.core" name="jackson-annotations" rev="${/com.fasterxml.jackson.core/jackson-annotations}" conf="test"/>

solr/core/src/java/org/apache/solr/cloud/rule/ImplicitSnitch.java

@@ -17,21 +17,29 @@
 package org.apache.solr.cloud.rule;
 
 import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.net.InetAddress;
 import java.nio.file.Files;
 import java.nio.file.Paths;
+import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
 import org.apache.solr.common.params.ModifiableSolrParams;
 import org.apache.solr.core.CoreContainer;
 import org.apache.solr.handler.admin.CoreAdminHandler;
 import org.apache.solr.request.SolrQueryRequest;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class ImplicitSnitch extends Snitch implements CoreAdminHandler.Invocable {
+  private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
 
   public static final Pattern hostAndPortPattern = Pattern.compile("(?:https?://)?([^:]+):(\\d+)");
 
@@ -42,8 +50,10 @@ public class ImplicitSnitch extends Snitch implements CoreAdminHandler.Invocable
   public static final String CORES = "cores";
   public static final String DISK = "freedisk";
   public static final String SYSPROP = "sysprop.";
+  public static final List<String> IP_SNITCHES = ImmutableList.of("ip_1", "ip_2", "ip_3", "ip_4");
+
+  public static final Set<String> tags = ImmutableSet.<String>builder().add(NODE, PORT, HOST, CORES, DISK).addAll(IP_SNITCHES).build();
 
-  public static final Set<String> tags = ImmutableSet.of(NODE, PORT, HOST, CORES, DISK);
 
 
   @Override
@@ -57,6 +67,9 @@ public class ImplicitSnitch extends Snitch implements CoreAdminHandler.Invocable
       Matcher hostAndPortMatcher = hostAndPortPattern.matcher(solrNode);
       if (hostAndPortMatcher.find()) ctx.getTags().put(PORT, hostAndPortMatcher.group(2));
     }
+
+    addIpTags(solrNode, requestedTags, ctx);
+
     ModifiableSolrParams params = new ModifiableSolrParams();
     if (requestedTags.contains(CORES)) params.add(CORES, "1");
     if (requestedTags.contains(DISK)) params.add(DISK, "1");
@@ -88,16 +101,75 @@ public class ImplicitSnitch extends Snitch implements CoreAdminHandler.Invocable
     }
     String[] sysProps = req.getParams().getParams(SYSPROP);
     if (sysProps != null && sysProps.length > 0) {
-      for (String prop : sysProps) result.put(SYSPROP+prop, System.getProperty(prop));
+      for (String prop : sysProps) result.put(SYSPROP + prop, System.getProperty(prop));
     }
     return result;
   }
 
+  private static final String HOST_FRAG_SEPARATOR_REGEX = "\\.";
 
   @Override
   public boolean isKnownTag(String tag) {
     return tags.contains(tag) ||
-        tag.startsWith(SYSPROP);//a system property
+        tag.startsWith(SYSPROP);
+  }
+
+  private void addIpTags(String solrNode, Set<String> requestedTags, SnitchContext context) {
+
+    List<String> requestedHostTags = new ArrayList<>();
+    for (String tag : requestedTags) {
+      if (IP_SNITCHES.contains(tag)) {
+        requestedHostTags.add(tag);
+      }
+    }
+
+    if (requestedHostTags.isEmpty()) {
+      return;
+    }
+
+    String[] ipFragments = getIpFragments(solrNode);
+
+    if (ipFragments == null) {
+      return;
+    }
+
+    int ipSnitchCount = IP_SNITCHES.size();
+    for (int i = 0; i < ipSnitchCount; i++) {
+      String currentTagValue = ipFragments[i];
+      String currentTagKey = IP_SNITCHES.get(ipSnitchCount - i - 1);
+
+      if (requestedHostTags.contains(currentTagKey)) {
+        context.getTags().put(currentTagKey, currentTagValue);
+      }
+
+    }
+
+  }
+
+  private String[] getIpFragments(String solrNode) {
+    Matcher hostAndPortMatcher = hostAndPortPattern.matcher(solrNode);
+    if (hostAndPortMatcher.find()) {
+      String host = hostAndPortMatcher.group(1);
+      if (host != null) {
+        String ip = getHostIp(host);
+        if (ip != null) {
+          return ip.split(HOST_FRAG_SEPARATOR_REGEX); //IPv6 support will be provided by SOLR-8523
+        }
+      }
+    }
+
+    log.warn("Failed to match host IP address from node URL [{}] using regex [{}]", solrNode, hostAndPortPattern.pattern());
+    return null;
+  }
+
+  protected String getHostIp(String host) {
+    try {
+      InetAddress address = InetAddress.getByName(host);
+      return address.getHostAddress();
+    } catch (Exception e) {
+      log.warn("Failed to get IP address from host [{}], with exception [{}] ", host, e);
+      return null;
+    }
   }
 
 }

solr/core/src/test/org/apache/solr/cloud/rule/ImplicitSnitchTest.java

@@ -0,0 +1,186 @@
+package org.apache.solr.cloud.rule;
+
+import java.util.Map;
+
+import com.google.common.collect.Sets;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import static org.hamcrest.core.Is.is;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Mockito.when;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+public class ImplicitSnitchTest {
+
+  private ImplicitSnitch snitch;
+  private SnitchContext context;
+
+  private static final String IP_1 = "ip_1";
+  private static final String IP_2 = "ip_2";
+  private static final String IP_3 = "ip_3";
+  private static final String IP_4 = "ip_4";
+
+  @Before
+  public void beforeImplicitSnitchTest() {
+    snitch = new ImplicitSnitch();
+    context = new SnitchContext(null, null);
+  }
+
+
+  @Test
+  public void testGetTags_withAllIPv4RequestedTags_with_omitted_zeros_returns_four_tags() throws Exception {
+    String node = "5:8983_solr";
+
+    snitch.getTags(node, Sets.newHashSet(IP_1, IP_2, IP_3, IP_4), context);
+
+    Map<String, Object> tags = context.getTags();
+    assertThat(tags.entrySet().size(), is(4));
+    assertThat(tags.get(IP_1), is("5"));
+    assertThat(tags.get(IP_2), is("0"));
+    assertThat(tags.get(IP_3), is("0"));
+    assertThat(tags.get(IP_4), is("0"));
+  }
+
+
+  @Test
+  public void testGetTags_withAllIPv4RequestedTags_returns_four_tags() throws Exception {
+    String node = "192.168.1.2:8983_solr";
+
+    snitch.getTags(node, Sets.newHashSet(IP_1, IP_2, IP_3, IP_4), context);
+
+    Map<String, Object> tags = context.getTags();
+    assertThat(tags.entrySet().size(), is(4));
+    assertThat(tags.get(IP_1), is("2"));
+    assertThat(tags.get(IP_2), is("1"));
+    assertThat(tags.get(IP_3), is("168"));
+    assertThat(tags.get(IP_4), is("192"));
+  }
+
+  @Test
+  public void testGetTags_withIPv4RequestedTags_ip2_and_ip4_returns_two_tags() throws Exception {
+    String node = "192.168.1.2:8983_solr";
+
+    SnitchContext context = new SnitchContext(null, node);
+    snitch.getTags(node, Sets.newHashSet(IP_2, IP_4), context);
+
+    Map<String, Object> tags = context.getTags();
+    assertThat(tags.entrySet().size(), is(2));
+    assertThat(tags.get(IP_2), is("1"));
+    assertThat(tags.get(IP_4), is("192"));
+  }
+
+  @Test
+  public void testGetTags_with_wrong_ipv4_format_ip_returns_nothing() throws Exception {
+    String node = "192.168.1.2.1:8983_solr";
+
+    SnitchContext context = new SnitchContext(null, node);
+    snitch.getTags(node, Sets.newHashSet(IP_1), context);
+
+    Map<String, Object> tags = context.getTags();
+    assertThat(tags.entrySet().size(), is(0));
+  }
+
+
+  @Test
+  public void testGetTags_with_correct_ipv6_format_ip_returns_nothing() throws Exception {
+    String node = "[0:0:0:0:0:0:0:1]:8983_solr";
+
+    SnitchContext context = new SnitchContext(null, node);
+    snitch.getTags(node, Sets.newHashSet(IP_1), context);
+
+    Map<String, Object> tags = context.getTags();
+    assertThat(tags.entrySet().size(), is(0)); //This will fail when IPv6 is implemented
+  }
+
+
+  @Test
+  public void testGetTags_withEmptyRequestedTag_returns_nothing() throws Exception {
+    String node = "192.168.1.2:8983_solr";
+
+    snitch.getTags(node, Sets.newHashSet(), context);
+
+    Map<String, Object> tags = context.getTags();
+    assertThat(tags.entrySet().size(), is(0));
+  }
+
+
+  @Test
+  public void testGetTags_withAllHostNameRequestedTags_returns_all_Tags() throws Exception {
+    String node = "serv01.dc01.london.uk.apache.org:8983_solr";
+
+    SnitchContext context = new SnitchContext(null, node);
+    //We need mocking here otherwise, we would need proper DNS entry for this test to pass
+    ImplicitSnitch mockedSnitch = Mockito.spy(snitch);
+    when(mockedSnitch.getHostIp(anyString())).thenReturn("10.11.12.13");
+
+    mockedSnitch.getTags(node, Sets.newHashSet(IP_1, IP_2, IP_3, IP_4), context);
+
+    Map<String, Object> tags = context.getTags();
+    assertThat(tags.entrySet().size(), is(4));
+    assertThat(tags.get(IP_1), is("13"));
+    assertThat(tags.get(IP_2), is("12"));
+    assertThat(tags.get(IP_3), is("11"));
+    assertThat(tags.get(IP_4), is("10"));
+  }
+
+  @Test
+  public void testGetTags_withHostNameRequestedTag_ip3_returns_1_tag() throws Exception {
+    String node = "serv01.dc01.london.uk.apache.org:8983_solr";
+
+    SnitchContext context = new SnitchContext(null, node);
+    //We need mocking here otherwise, we would need proper DNS entry for this test to pass
+    ImplicitSnitch mockedSnitch = Mockito.spy(snitch);
+    when(mockedSnitch.getHostIp(anyString())).thenReturn("10.11.12.13");
+    mockedSnitch.getTags(node, Sets.newHashSet(IP_3), context);
+
+    Map<String, Object> tags = context.getTags();
+    assertThat(tags.entrySet().size(), is(1));
+    assertThat(tags.get(IP_3), is("11"));
+  }
+
+  @Test
+  public void testGetTags_withHostNameRequestedTag_ip99999_returns_nothing() throws Exception {
+    String node = "serv01.dc01.london.uk.apache.org:8983_solr";
+
+    SnitchContext context = new SnitchContext(null, node);
+    //We need mocking here otherwise, we would need proper DNS entry for this test to pass
+    ImplicitSnitch mockedSnitch = Mockito.spy(snitch);
+    when(mockedSnitch.getHostIp(anyString())).thenReturn("10.11.12.13");
+    mockedSnitch.getTags(node, Sets.newHashSet("ip_99999"), context);
+
+    Map<String, Object> tags = context.getTags();
+    assertThat(tags.entrySet().size(), is(0));
+  }
+
+  @Test
+  public void testIsKnownTag_ip1() throws Exception {
+    assertFalse(snitch.isKnownTag("ip_0"));
+    assertTrue(snitch.isKnownTag(IP_1));
+    assertTrue(snitch.isKnownTag(IP_2));
+    assertTrue(snitch.isKnownTag(IP_3));
+    assertTrue(snitch.isKnownTag(IP_4));
+    assertFalse(snitch.isKnownTag("ip_5"));
+  }
+
+}

solr/core/src/test/org/apache/solr/cloud/rule/RulesTest.java

@@ -34,15 +34,20 @@ import org.apache.solr.common.cloud.ImplicitDocRouter;
 import org.apache.solr.common.cloud.ZkStateReader;
 import org.apache.solr.common.params.ModifiableSolrParams;
 import org.junit.Test;
+import org.junit.rules.ExpectedException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import static org.apache.solr.client.solrj.SolrRequest.METHOD.POST;
 import static org.apache.solr.common.params.CommonParams.COLLECTIONS_HANDLER_PATH;
+import static org.junit.matchers.JUnitMatchers.containsString;
 
 public class RulesTest extends AbstractFullDistribZkTestBase {
   private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
 
+  @org.junit.Rule
+  public ExpectedException expectedException = ExpectedException.none();
+
   @Test
   @ShardsFixed(num = 5)
   public void doIntegrationTest() throws Exception {
@@ -127,6 +132,78 @@ public class RulesTest extends AbstractFullDistribZkTestBase {
     assertEquals ( "ImplicitSnitch", ((Map)list.get(0)).get("class"));
   }
 
+  @Test
+  public void testHostFragmentRule() throws Exception {
+    String rulesColl = "ipRuleColl";
+    String baseUrl = getBaseUrl((HttpSolrClient) clients.get(0));
+    String ip_1 = "-1";
+    String ip_2 = "-1";
+    Matcher hostAndPortMatcher = Pattern.compile("(?:https?://)?([^:]+):(\\d+)").matcher(baseUrl);
+    if (hostAndPortMatcher.find()) {
+      String[] ipFragments = hostAndPortMatcher.group(1).split("\\.");
+      ip_1 = ipFragments[ipFragments.length - 1];
+      ip_2 = ipFragments[ipFragments.length - 2];
+    }
+
+    try (SolrClient client = createNewSolrClient("", baseUrl)) {
+      CollectionAdminResponse rsp;
+      CollectionAdminRequest.Create create = new CollectionAdminRequest.Create();
+      create.setCollectionName(rulesColl);
+      create.setShards("shard1");
+      create.setRouterName(ImplicitDocRouter.NAME);
+      create.setReplicationFactor(2);
+      create.setRule("ip_2:" + ip_2, "ip_1:" + ip_1);
+      create.setSnitch("class:ImplicitSnitch");
+      rsp = create.process(client);
+      assertEquals(0, rsp.getStatus());
+      assertTrue(rsp.isSuccess());
+
+    }
+
+    DocCollection rulesCollection = cloudClient.getZkStateReader().getClusterState().getCollection(rulesColl);
+    List<Map> list = (List<Map>) rulesCollection.get("rule");
+    assertEquals(2, list.size());
+    assertEquals(ip_2, list.get(0).get("ip_2"));
+    assertEquals(ip_1, list.get(1).get("ip_1"));
+
+    list = (List) rulesCollection.get("snitch");
+    assertEquals(1, list.size());
+    assertEquals("ImplicitSnitch", list.get(0).get("class"));
+  }
+
+
+  @Test
+  public void testHostFragmentRuleThrowsExceptionWhenIpDoesNotMatch() throws Exception {
+    String rulesColl = "ipRuleColl";
+    String baseUrl = getBaseUrl((HttpSolrClient) clients.get(0));
+    String ip_1 = "-1";
+    String ip_2 = "-1";
+    Matcher hostAndPortMatcher = Pattern.compile("(?:https?://)?([^:]+):(\\d+)").matcher(baseUrl);
+    if (hostAndPortMatcher.find()) {
+      String[] ipFragments = hostAndPortMatcher.group(1).split("\\.");
+      ip_1 = ipFragments[ipFragments.length - 1];
+      ip_2 = ipFragments[ipFragments.length - 2];
+    }
+
+    try (SolrClient client = createNewSolrClient("", baseUrl)) {
+      CollectionAdminRequest.Create create = new CollectionAdminRequest.Create();
+      create.setCollectionName(rulesColl);
+      create.setShards("shard1");
+      create.setRouterName(ImplicitDocRouter.NAME);
+      create.setReplicationFactor(2);
+
+      create.setRule("ip_2:" + ip_2, "ip_1:" + ip_1 + "9999");
+      create.setSnitch("class:ImplicitSnitch");
+
+      expectedException.expect(HttpSolrClient.RemoteSolrException.class);
+      expectedException.expectMessage(containsString("ip_1"));
+
+      create.process(client);
+    }
+
+  }
+
+
   @Test
   public void testModifyColl() throws Exception {
     final long minGB1 = (random().nextBoolean() ? 1 : 0);