IP Filtering: Add support for having on filters on HTTP transport

In order to fix elastic/elasticsearch#378 a problem was revealed, that the ip filter for HTTP was
always the one for the default profile, which lead to failed tests (along
with wrong socket connections, which made the test go green irregularly).

This commit fixes the tests and allow to configure own HTTP ip filters, adding
the following settings

* shield.http.filter.enabled
* shield.http.filter.allow
* shield.http.filter.deny

If not specific settings are configured, the one of the default profile are used.

Closes elastic/elasticsearch#378

Original commit: elastic/x-pack-elasticsearch@89dbaefe9a

src/main/java/org/elasticsearch/shield/transport/filter/IPFilter.java

@@ -27,6 +27,14 @@ import java.util.Map;
 
 public class IPFilter extends AbstractComponent {
 
+    /**
+     * .http has been chosen for handling HTTP filters, which are not part of the profiles
+     * The profiles are only handled for the transport protocol, so we need an own kind of profile
+     * for HTTP. This name starts withs a dot, because no profile name can ever start like that due to
+     * how we handle settings
+     */
+    public static final String HTTP_PROFILE_NAME = ".http";
+
     private static final ProfileIpFilterRule[] NO_RULES = new ProfileIpFilterRule[0];
     private static final ProfileIpFilterRule ACCEPT_ALL_RULE = new ProfileIpFilterRule("default",
             new PatternRule(true, "n:*"), "DEFAULT_ACCEPT_ALL");
@@ -46,6 +54,7 @@ public class IPFilter extends AbstractComponent {
         if (rules == NO_RULES) {
             return true;
         }
+
         for (ProfileIpFilterRule rule : rules) {
             if (rule.contains(profile, peerAddress)) {
                 boolean isAllowed = rule.isAllowRule();
@@ -68,11 +77,15 @@ public class IPFilter extends AbstractComponent {
         }
         String[] allowed = settings.getAsArray("shield.transport.filter.allow");
         String[] denied = settings.getAsArray("shield.transport.filter.deny");
+        String[] httpAllowed = settings.getAsArray("shield.http.filter.allow", settings.getAsArray("transport.profiles.default.shield.filter.allow", settings.getAsArray("shield.transport.filter.allow")));
+        String[] httpDdenied = settings.getAsArray("shield.http.filter.deny", settings.getAsArray("transport.profiles.default.shield.filter.deny", settings.getAsArray("shield.transport.filter.deny")));
         List<ProfileIpFilterRule> rules = new ArrayList<>();
 
         try {
             rules.addAll(parseValue(allowed, "default", true));
             rules.addAll(parseValue(denied, "default", false));
+            rules.addAll(parseValue(httpAllowed, HTTP_PROFILE_NAME, true));
+            rules.addAll(parseValue(httpDdenied, HTTP_PROFILE_NAME, false));
 
             Map<String, Settings> groupedSettings = settings.getGroups("transport.profiles.");
             for (Map.Entry<String, Settings> entry : groupedSettings.entrySet()) {
@@ -109,5 +122,4 @@ public class IPFilter extends AbstractComponent {
         String prefix = isInetAddress ? "i:" : "n:";
         return new PatternRule(isAllowRule, prefix + value);
     }
-
 }

src/main/java/org/elasticsearch/shield/transport/netty/NettySecuredHttpServerTransport.java

@@ -57,7 +57,7 @@ public class NettySecuredHttpServerTransport extends NettyHttpServerTransport {
 
                 pipeline.addFirst("ssl", new SslHandler(engine));
             }
-            pipeline.addFirst("ipfilter", new NettyIPFilterUpstreamHandler(ipFilter, "default"));
+            pipeline.addFirst("ipfilter", new NettyIPFilterUpstreamHandler(ipFilter, IPFilter.HTTP_PROFILE_NAME));
             return pipeline;
         }
     }

src/test/java/org/elasticsearch/shield/transport/filter/IPFilterTests.java

@@ -122,6 +122,32 @@ public class IPFilterTests extends ElasticsearchTestCase {
         assertAddressIsAllowed("10.0.0.2");
     }
 
+    @Test
+    public void testThatHttpWorks() throws Exception {
+        Settings settings = settingsBuilder()
+                .put("shield.transport.filter.allow", "127.0.0.1")
+                .put("shield.transport.filter.deny", "10.0.0.0/8")
+                .put("shield.http.filter.allow", "10.0.0.0/8")
+                .put("shield.http.filter.deny", "127.0.0.1")
+                .build();
+        ipFilter = new IPFilter(settings, auditTrail);
+
+        assertAddressIsAllowedForProfile(IPFilter.HTTP_PROFILE_NAME, "10.2.3.4");
+        assertAddressIsDeniedForProfile(IPFilter.HTTP_PROFILE_NAME, "127.0.0.1");
+    }
+
+    @Test
+    public void testThatHttpFallsbackToDefault() throws Exception {
+        Settings settings = settingsBuilder()
+                .put("shield.transport.filter.allow", "127.0.0.1")
+                .put("shield.transport.filter.deny", "10.0.0.0/8")
+                .build();
+        ipFilter = new IPFilter(settings, auditTrail);
+
+        assertAddressIsAllowedForProfile(IPFilter.HTTP_PROFILE_NAME, "127.0.0.1");
+        assertAddressIsDeniedForProfile(IPFilter.HTTP_PROFILE_NAME, "10.2.3.4");
+    }
+
     private void assertAddressIsAllowedForProfile(String profile, String ... inetAddresses) {
         for (String inetAddress : inetAddresses) {
             String message = String.format(Locale.ROOT, "Expected address %s to be allowed", inetAddress);

src/test/java/org/elasticsearch/shield/transport/filter/IpFilteringIntegrationTests.java

@@ -6,7 +6,7 @@
 package org.elasticsearch.shield.transport.filter;
 
 import com.google.common.base.Charsets;
-import org.apache.lucene.util.LuceneTestCase;
+import org.elasticsearch.client.Client;
 import org.elasticsearch.common.settings.ImmutableSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.InetSocketTransportAddress;
@@ -14,62 +14,75 @@ import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.http.HttpServerTransport;
 import org.elasticsearch.node.internal.InternalNode;
 import org.elasticsearch.test.ShieldIntegrationTest;
-import org.elasticsearch.transport.Transport;
+import org.junit.BeforeClass;
 import org.junit.Test;
 
-import java.io.InputStream;
+import java.io.IOException;
 import java.io.OutputStream;
 import java.net.InetSocketAddress;
 import java.net.Socket;
-import java.net.SocketException;
 
 import static org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope;
 import static org.elasticsearch.test.ElasticsearchIntegrationTest.Scope;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
 
-@LuceneTestCase.AwaitsFix(bugUrl = "https://github.com/elasticsearch/elasticsearch-shield/issues/378")
 // no client nodes, no transport clients, as they all get rejected on network connections
 @ClusterScope(scope = Scope.SUITE, numDataNodes = 1, numClientNodes = 0, transportClientRatio = 0.0)
 public class IpFilteringIntegrationTests extends ShieldIntegrationTest {
 
-    @Override
+    private static int randomClientPort;
-    protected Settings nodeSettings(int nodeOrdinal) {
+
-        return ImmutableSettings.builder().put(super.nodeSettings(nodeOrdinal))
+    @BeforeClass
-                .put(InternalNode.HTTP_ENABLED, true)
+    public static void getRandomPort() {
-                .put("shield.transport.filter.deny", "_all").build();
+        randomClientPort = randomIntBetween(49000, 65500); // ephemeral port
     }
 
-    @Test(expected = SocketException.class)
+    @Override
+    protected Settings nodeSettings(int nodeOrdinal) {
+        String randomClientPortRange = randomClientPort + "-" + (randomClientPort+100);
+        return ImmutableSettings.builder().put(super.nodeSettings(nodeOrdinal))
+                .put(InternalNode.HTTP_ENABLED, true)
+                .put("transport.profiles.client.port", randomClientPortRange)
+                .put("transport.profiles.client.bind_host", "localhost") // make sure this is "localhost", no matter if ipv4 or ipv6, but be consistent
+                .put("transport.profiles.client.shield.filter.deny", "_all")
+                .put("shield.http.filter.deny", "_all").build();
+    }
+
+    @Test
     public void testThatIpFilteringIsIntegratedIntoNettyPipelineViaHttp() throws Exception {
         TransportAddress transportAddress = internalCluster().getDataNodeInstance(HttpServerTransport.class).boundAddress().boundAddress();
         assertThat(transportAddress, is(instanceOf(InetSocketTransportAddress.class)));
         InetSocketTransportAddress inetSocketTransportAddress = (InetSocketTransportAddress) transportAddress;
 
-        trySocketConnection(inetSocketTransportAddress.address());
-    }
-
-    @Test(expected = SocketException.class)
-    public void testThatIpFilteringIsIntegratedIntoNettyPipelineViaTransportClient() throws Exception {
-        TransportAddress transportAddress = internalCluster().getDataNodeInstance(Transport.class).boundAddress().boundAddress();
-        assertThat(transportAddress, is(instanceOf(InetSocketTransportAddress.class)));
-        InetSocketTransportAddress inetSocketTransportAddress = (InetSocketTransportAddress) transportAddress;
-        trySocketConnection(inetSocketTransportAddress.address());
-    }
-
-    private void trySocketConnection(InetSocketAddress address) throws Exception {
         try (Socket socket = new Socket()){
+            trySocketConnection(socket, inetSocketTransportAddress.address());
+            assertThat(socket.isClosed(), is(true));
+        }
+    }
+
+    @Test
+    public void testThatIpFilteringIsNotAppliedForDefaultTransport() throws Exception {
+        Client client = internalCluster().transportClient();
+        assertGreenClusterState(client);
+    }
+
+    @Test
+    public void testThatIpFilteringIsAppliedForProfile() throws Exception {
+        try (Socket socket = new Socket()){
+            trySocketConnection(socket, new InetSocketAddress("localhost", randomClientPort));
+            assertThat(socket.isClosed(), is(true));
+        }
+    }
+
+    private void trySocketConnection(Socket socket, InetSocketAddress address) throws IOException {
         logger.info("Connecting to {}", address);
         socket.connect(address, 500);
 
         assertThat(socket.isConnected(), is(true));
         try (OutputStream os = socket.getOutputStream()) {
-                os.write("foo".getBytes(Charsets.UTF_8));
+            os.write("fooooo".getBytes(Charsets.UTF_8));
             os.flush();
         }
-            try (InputStream is = socket.getInputStream()) {
-                is.read();
-            }
-        }
     }
 }

src/test/java/org/elasticsearch/shield/transport/netty/NettyIPFilterUpstreamHandlerTests.java

@@ -36,7 +36,7 @@ public class NettyIPFilterUpstreamHandlerTests extends ElasticsearchTestCase {
 
         IPFilter ipFilter = new IPFilter(settings, AuditTrail.NOOP);
 
-        nettyUpstreamHandler = new NettyIPFilterUpstreamHandler(ipFilter, "default");
+        nettyUpstreamHandler = new NettyIPFilterUpstreamHandler(ipFilter, IPFilter.HTTP_PROFILE_NAME);
     }
 
     @Test