Merge branch 'master' into feature/query-refactoring

core/src/main/java/org/elasticsearch/action/search/SearchType.java

@@ -54,7 +54,9 @@ public enum SearchType {
     /**
      * Performs scanning of the results which executes the search without any sorting.
      * It will automatically start scrolling the result set.
+     * @deprecated will be removed in 3.0, you should do a regular scroll instead, ordered by `_doc`
      */
+    @Deprecated
     SCAN((byte) 4),
     /**
      * Only counts the results, will still execute aggregations and the like.
@@ -69,6 +71,7 @@ public enum SearchType {
     public static final SearchType DEFAULT = QUERY_THEN_FETCH;
 
     private static final ParseField COUNT_VALUE = new ParseField("count").withAllDeprecated("query_then_fetch");
+    private static final ParseField SCAN_VALUE = new ParseField("scan").withAllDeprecated("query_then_fetch sorting on `_doc`");
 
     private byte id;
 
@@ -121,7 +124,7 @@ public enum SearchType {
             return SearchType.QUERY_THEN_FETCH;
         } else if ("query_and_fetch".equals(searchType)) {
             return SearchType.QUERY_AND_FETCH;
-        } else if ("scan".equals(searchType)) {
+        } else if (parseFieldMatcher.match(searchType, SCAN_VALUE)) {
             return SearchType.SCAN;
         } else if (parseFieldMatcher.match(searchType, COUNT_VALUE)) {
             return SearchType.COUNT;

core/src/main/java/org/elasticsearch/action/search/type/TransportSearchScanAction.java

@@ -40,6 +40,7 @@ import org.elasticsearch.threadpool.ThreadPool;
 
 import static org.elasticsearch.action.search.type.TransportSearchHelper.buildScrollId;
 
+@Deprecated // remove in 3.0
 public class TransportSearchScanAction extends TransportSearchTypeAction {
 
     @Inject

core/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java

@@ -291,7 +291,7 @@ public class Bootstrap {
                 Loggers.enableConsoleLogging();
             }
             
-            throw e;
+            throw new StartupError(e);
         }
     }
 

core/src/main/java/org/elasticsearch/bootstrap/StartupError.java

@@ -0,0 +1,118 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.bootstrap;
+
+import org.elasticsearch.common.inject.CreationException;
+import org.elasticsearch.common.inject.spi.Message;
+
+import java.io.PrintStream;
+
+/**
+ * Wraps an exception in a special way that it gets formatted
+ * "reasonably". This means limits on stacktrace frames and
+ * cleanup for guice, and some guidance about consulting full
+ * logs for the whole exception.
+ */
+//TODO: remove this when guice is removed, and exceptions are cleaned up
+//this is horrible, but its what we must do
+class StartupError extends RuntimeException {
+    
+    /** maximum length of a stacktrace, before we truncate it */
+    static final int STACKTRACE_LIMIT = 30;
+    /** all lines from this package are RLE-compressed */
+    static final String GUICE_PACKAGE = "org.elasticsearch.common.inject";
+    
+    /** 
+     * Create a new StartupError that will format {@code cause}
+     * to the console on failure.
+     */
+    StartupError(Throwable cause) {
+        super(cause);
+    }
+
+    /*
+     * This logic actually prints the exception to the console, its
+     * what is invoked by the JVM when we throw the exception from main()
+     */
+    @Override
+    public void printStackTrace(PrintStream s) {
+        Throwable originalCause = getCause();
+        Throwable cause = originalCause;
+        if (cause instanceof CreationException) {
+            cause = getFirstGuiceCause((CreationException)cause);
+        }
+        
+        String message = cause.getMessage();
+        if (message == null) {
+            message = "Unknown Error";
+        }
+        s.println(message);
+        
+        if (cause != null) {
+            // walk to the root cause
+            while (cause.getCause() != null) {
+                cause = cause.getCause();
+            }
+
+            // print the root cause message, only if it differs!
+            if (cause != originalCause && (message.equals(cause.toString()) == false)) {
+                s.println("Likely root cause: " + cause);
+            }
+
+            // print stacktrace of cause
+            StackTraceElement stack[] = cause.getStackTrace();
+            int linesWritten = 0;
+            for (int i = 0; i < stack.length; i++) {
+                if (linesWritten == STACKTRACE_LIMIT) {
+                    s.println("\t<<<truncated>>>");
+                    break;
+                }
+                String line = stack[i].toString();
+                
+                // skip past contiguous runs of this garbage:
+                if (line.startsWith(GUICE_PACKAGE)) {
+                    while (i + 1 < stack.length && stack[i + 1].toString().startsWith(GUICE_PACKAGE)) {
+                        i++;
+                    }
+                    s.println("\tat <<<guice>>>");
+                    linesWritten++;
+                    continue;
+                }
+
+                s.println("\tat " + line.toString());
+                linesWritten++;
+            }
+        }
+        s.println("Refer to the log for complete error details.");
+    }
+    
+    /** 
+     * Returns first cause from a guice error (it can have multiple).
+     */
+    static Throwable getFirstGuiceCause(CreationException guice) {
+        for (Message message : guice.getErrorMessages()) {
+            Throwable cause = message.getCause();
+            if (cause != null) {
+                return cause;
+            }
+        }
+        return guice; // we tried
+    }
+}

core/src/main/java/org/elasticsearch/cluster/ClusterInfo.java

@@ -30,7 +30,7 @@ import java.util.Map;
  * <code>InternalClusterInfoService.shardIdentifierFromRouting(String)</code>
  * for the key used in the shardSizes map
  */
-public final class ClusterInfo {
+public class ClusterInfo {
 
     private final Map<String, DiskUsage> usages;
     final Map<String, Long> shardSizes;
@@ -54,6 +54,11 @@ public final class ClusterInfo {
         return shardSizes.get(shardIdentifierFromRouting(shardRouting));
     }
 
+    public long getShardSize(ShardRouting shardRouting, long defaultValue) {
+        Long shardSize = getShardSize(shardRouting);
+        return shardSize == null ? defaultValue : shardSize;
+    }
+
     /**
      * Method that incorporates the ShardId for the shard into a string that
      * includes a 'p' or 'r' depending on whether the shard is a primary.

core/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java

@@ -19,6 +19,7 @@
 
 package org.elasticsearch.cluster;
 
+import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.LatchedActionListener;
 import org.elasticsearch.action.admin.cluster.node.stats.NodeStats;
@@ -36,6 +37,7 @@ import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
 import org.elasticsearch.monitor.fs.FsInfo;
 import org.elasticsearch.node.settings.NodeSettingsService;
@@ -45,6 +47,7 @@ import org.elasticsearch.transport.ReceiveTimeoutTransportException;
 import java.util.*;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
 
 /**
  * InternalClusterInfoService provides the ClusterInfoService interface,

core/src/main/java/org/elasticsearch/cluster/node/DiscoveryNode.java

@@ -360,16 +360,16 @@ public class DiscoveryNode implements Streamable, ToXContent {
     public String toString() {
         StringBuilder sb = new StringBuilder();
         if (nodeName.length() > 0) {
-            sb.append('[').append(nodeName).append(']');
+            sb.append('{').append(nodeName).append('}');
         }
         if (nodeId != null) {
-            sb.append('[').append(nodeId).append(']');
+            sb.append('{').append(nodeId).append('}');
         }
         if (Strings.hasLength(hostName)) {
-            sb.append('[').append(hostName).append(']');
+            sb.append('{').append(hostName).append('}');
         }
         if (address != null) {
-            sb.append('[').append(address).append(']');
+            sb.append('{').append(address).append('}');
         }
         if (!attributes.isEmpty()) {
             sb.append(attributes);

core/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java

@@ -345,10 +345,10 @@ public class RoutingNodes implements Iterable<RoutingNode> {
     /**
      * Moves a shard from unassigned to initialize state
      */
-    public void initialize(ShardRouting shard, String nodeId) {
+    public void initialize(ShardRouting shard, String nodeId, long expectedSize) {
         ensureMutable();
         assert shard.unassigned() : shard;
-        shard.initialize(nodeId);
+        shard.initialize(nodeId, expectedSize);
         node(nodeId).add(shard);
         inactiveShardCount++;
         if (shard.primary()) {
@@ -362,10 +362,10 @@ public class RoutingNodes implements Iterable<RoutingNode> {
      * shard as well as assigning it. And returning the target initializing
      * shard.
      */
-    public ShardRouting relocate(ShardRouting shard, String nodeId) {
+    public ShardRouting relocate(ShardRouting shard, String nodeId, long expectedShardSize) {
         ensureMutable();
         relocatingShards++;
-        shard.relocate(nodeId);
+        shard.relocate(nodeId, expectedShardSize);
         ShardRouting target = shard.buildTargetRelocatingShard();
         node(target.currentNodeId()).add(target);
         assignedShardsAdd(target);
@@ -608,16 +608,9 @@ public class RoutingNodes implements Iterable<RoutingNode> {
             /**
              * Initializes the current unassigned shard and moves it from the unassigned list.
              */
-            public void initialize(String nodeId) {
-                initialize(nodeId, current.version());
-            }
-
-            /**
-             * Initializes the current unassigned shard and moves it from the unassigned list.
-             */
-            public void initialize(String nodeId, long version) {
+            public void initialize(String nodeId, long version, long expectedShardSize) {
                 innerRemove();
-                nodes.initialize(new ShardRouting(current, version), nodeId);
+                nodes.initialize(new ShardRouting(current, version), nodeId, expectedShardSize);
             }
 
             /**

core/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java

@@ -37,6 +37,11 @@ import java.util.List;
  */
 public final class ShardRouting implements Streamable, ToXContent {
 
+    /**
+     * Used if shard size is not available
+     */
+    public static final long UNAVAILABLE_EXPECTED_SHARD_SIZE = -1;
+
     private String index;
     private int shardId;
     private String currentNodeId;
@@ -50,6 +55,7 @@ public final class ShardRouting implements Streamable, ToXContent {
     private final transient List<ShardRouting> asList;
     private transient ShardId shardIdentifier;
     private boolean frozen = false;
+    private long expectedShardSize = UNAVAILABLE_EXPECTED_SHARD_SIZE;
 
     private ShardRouting() {
         this.asList = Collections.singletonList(this);
@@ -60,7 +66,7 @@ public final class ShardRouting implements Streamable, ToXContent {
     }
 
     public ShardRouting(ShardRouting copy, long version) {
-        this(copy.index(), copy.id(), copy.currentNodeId(), copy.relocatingNodeId(), copy.restoreSource(), copy.primary(), copy.state(), version, copy.unassignedInfo(), copy.allocationId(), true);
+        this(copy.index(), copy.id(), copy.currentNodeId(), copy.relocatingNodeId(), copy.restoreSource(), copy.primary(), copy.state(), version, copy.unassignedInfo(), copy.allocationId(), true, copy.getExpectedShardSize());
     }
 
     /**
@@ -69,7 +75,7 @@ public final class ShardRouting implements Streamable, ToXContent {
      */
     ShardRouting(String index, int shardId, String currentNodeId,
                  String relocatingNodeId, RestoreSource restoreSource, boolean primary, ShardRoutingState state, long version,
-                 UnassignedInfo unassignedInfo, AllocationId allocationId, boolean internal) {
+                 UnassignedInfo unassignedInfo, AllocationId allocationId, boolean internal, long expectedShardSize) {
         this.index = index;
         this.shardId = shardId;
         this.currentNodeId = currentNodeId;
@@ -81,6 +87,9 @@ public final class ShardRouting implements Streamable, ToXContent {
         this.restoreSource = restoreSource;
         this.unassignedInfo = unassignedInfo;
         this.allocationId = allocationId;
+        this.expectedShardSize = expectedShardSize;
+        assert expectedShardSize == UNAVAILABLE_EXPECTED_SHARD_SIZE || state == ShardRoutingState.INITIALIZING || state == ShardRoutingState.RELOCATING : expectedShardSize + " state: " + state;
+        assert expectedShardSize >= 0 || state != ShardRoutingState.INITIALIZING || state != ShardRoutingState.RELOCATING : expectedShardSize + " state: " + state;
         assert !(state == ShardRoutingState.UNASSIGNED && unassignedInfo == null) : "unassigned shard must be created with meta";
         if (!internal) {
             assert state == ShardRoutingState.UNASSIGNED;
@@ -88,13 +97,14 @@ public final class ShardRouting implements Streamable, ToXContent {
             assert relocatingNodeId == null;
             assert allocationId == null;
         }
+
     }
 
     /**
      * Creates a new unassigned shard.
      */
     public static ShardRouting newUnassigned(String index, int shardId, RestoreSource restoreSource, boolean primary, UnassignedInfo unassignedInfo) {
-        return new ShardRouting(index, shardId, null, null, restoreSource, primary, ShardRoutingState.UNASSIGNED, 0, unassignedInfo, null, true);
+        return new ShardRouting(index, shardId, null, null, restoreSource, primary, ShardRoutingState.UNASSIGNED, 0, unassignedInfo, null, true, UNAVAILABLE_EXPECTED_SHARD_SIZE);
     }
 
     /**
@@ -205,7 +215,7 @@ public final class ShardRouting implements Streamable, ToXContent {
     public ShardRouting buildTargetRelocatingShard() {
         assert relocating();
         return new ShardRouting(index, shardId, relocatingNodeId, currentNodeId, restoreSource, primary, ShardRoutingState.INITIALIZING, version, unassignedInfo,
-                AllocationId.newTargetRelocation(allocationId), true);
+                AllocationId.newTargetRelocation(allocationId), true, expectedShardSize);
     }
 
     /**
@@ -317,6 +327,11 @@ public final class ShardRouting implements Streamable, ToXContent {
         if (in.readBoolean()) {
             allocationId = new AllocationId(in);
         }
+        if (relocating() || initializing()) {
+            expectedShardSize = in.readLong();
+        } else {
+            expectedShardSize = UNAVAILABLE_EXPECTED_SHARD_SIZE;
+        }
         freeze();
     }
 
@@ -368,6 +383,10 @@ public final class ShardRouting implements Streamable, ToXContent {
         } else {
             out.writeBoolean(false);
         }
+        if (relocating() || initializing()) {
+            out.writeLong(expectedShardSize);
+        }
+
     }
 
     @Override
@@ -397,12 +416,13 @@ public final class ShardRouting implements Streamable, ToXContent {
         relocatingNodeId = null;
         this.unassignedInfo = unassignedInfo;
         allocationId = null;
+        expectedShardSize = UNAVAILABLE_EXPECTED_SHARD_SIZE;
     }
 
     /**
      * Initializes an unassigned shard on a node.
      */
-    void initialize(String nodeId) {
+    void initialize(String nodeId, long expectedShardSize) {
         ensureNotFrozen();
         version++;
         assert state == ShardRoutingState.UNASSIGNED : this;
@@ -410,6 +430,7 @@ public final class ShardRouting implements Streamable, ToXContent {
         state = ShardRoutingState.INITIALIZING;
         currentNodeId = nodeId;
         allocationId = AllocationId.newInitializing();
+        this.expectedShardSize = expectedShardSize;
     }
 
     /**
@@ -417,13 +438,14 @@ public final class ShardRouting implements Streamable, ToXContent {
      *
      * @param relocatingNodeId id of the node to relocate the shard
      */
-    void relocate(String relocatingNodeId) {
+    void relocate(String relocatingNodeId, long expectedShardSize) {
         ensureNotFrozen();
         version++;
         assert state == ShardRoutingState.STARTED : "current shard has to be started in order to be relocated " + this;
         state = ShardRoutingState.RELOCATING;
         this.relocatingNodeId = relocatingNodeId;
         this.allocationId = AllocationId.newRelocation(allocationId);
+        this.expectedShardSize = expectedShardSize;
     }
 
     /**
@@ -436,7 +458,7 @@ public final class ShardRouting implements Streamable, ToXContent {
         assert state == ShardRoutingState.RELOCATING : this;
         assert assignedToNode() : this;
         assert relocatingNodeId != null : this;
-
+        expectedShardSize = UNAVAILABLE_EXPECTED_SHARD_SIZE;
         state = ShardRoutingState.STARTED;
         relocatingNodeId = null;
         allocationId = AllocationId.cancelRelocation(allocationId);
@@ -470,6 +492,7 @@ public final class ShardRouting implements Streamable, ToXContent {
             // relocation target
             allocationId = AllocationId.finishRelocation(allocationId);
         }
+        expectedShardSize = UNAVAILABLE_EXPECTED_SHARD_SIZE;
         state = ShardRoutingState.STARTED;
     }
 
@@ -669,6 +692,9 @@ public final class ShardRouting implements Streamable, ToXContent {
         if (this.unassignedInfo != null) {
             sb.append(", ").append(unassignedInfo.toString());
         }
+        if (expectedShardSize != UNAVAILABLE_EXPECTED_SHARD_SIZE) {
+            sb.append(", expected_shard_size[").append(expectedShardSize).append("]");
+        }
         return sb.toString();
     }
 
@@ -682,7 +708,9 @@ public final class ShardRouting implements Streamable, ToXContent {
                 .field("shard", shardId().id())
                 .field("index", shardId().index().name())
                 .field("version", version);
-
+        if (expectedShardSize != UNAVAILABLE_EXPECTED_SHARD_SIZE){
+            builder.field("expected_shard_size_in_bytes", expectedShardSize);
+        }
         if (restoreSource() != null) {
             builder.field("restore_source");
             restoreSource().toXContent(builder, params);
@@ -709,4 +737,12 @@ public final class ShardRouting implements Streamable, ToXContent {
     boolean isFrozen() {
         return frozen;
     }
+
+    /**
+     * Returns the expected shard size for {@link ShardRoutingState#RELOCATING} and {@link ShardRoutingState#INITIALIZING}
+     * shards. If it's size is not available {@value #UNAVAILABLE_EXPECTED_SHARD_SIZE} will be returned.
+     */
+    public long getExpectedShardSize() {
+        return expectedShardSize;
+    }
 }

core/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java

@@ -507,7 +507,7 @@ public class BalancedShardsAllocator extends AbstractComponent implements Shards
                     Decision decision = allocation.deciders().canAllocate(shard, target, allocation);
                     if (decision.type() == Type.YES) { // TODO maybe we can respect throttling here too?
                         sourceNode.removeShard(shard);
-                        ShardRouting targetRelocatingShard = routingNodes.relocate(shard, target.nodeId());
+                        ShardRouting targetRelocatingShard = routingNodes.relocate(shard, target.nodeId(), allocation.clusterInfo().getShardSize(shard, ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE));
                         currentNode.addShard(targetRelocatingShard, decision);
                         if (logger.isTraceEnabled()) {
                             logger.trace("Moved shard [{}] to node [{}]", shard, currentNode.getNodeId());
@@ -687,7 +687,7 @@ public class BalancedShardsAllocator extends AbstractComponent implements Shards
                             if (logger.isTraceEnabled()) {
                                 logger.trace("Assigned shard [{}] to [{}]", shard, minNode.getNodeId());
                             }
-                            routingNodes.initialize(shard, routingNodes.node(minNode.getNodeId()).nodeId());
+                            routingNodes.initialize(shard, routingNodes.node(minNode.getNodeId()).nodeId(), allocation.clusterInfo().getShardSize(shard, ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE));
                             changed = true;
                             continue; // don't add to ignoreUnassigned
                         } else {
@@ -779,10 +779,10 @@ public class BalancedShardsAllocator extends AbstractComponent implements Shards
                         /* now allocate on the cluster - if we are started we need to relocate the shard */
                         if (candidate.started()) {
                             RoutingNode lowRoutingNode = routingNodes.node(minNode.getNodeId());
-                            routingNodes.relocate(candidate, lowRoutingNode.nodeId());
+                            routingNodes.relocate(candidate, lowRoutingNode.nodeId(), allocation.clusterInfo().getShardSize(candidate, ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE));
 
                         } else {
-                            routingNodes.initialize(candidate, routingNodes.node(minNode.getNodeId()).nodeId());
+                            routingNodes.initialize(candidate, routingNodes.node(minNode.getNodeId()).nodeId(), allocation.clusterInfo().getShardSize(candidate, ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE));
                         }
                         return true;
 

core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateAllocationCommand.java

@@ -231,7 +231,7 @@ public class AllocateAllocationCommand implements AllocationCommand {
                 unassigned.updateUnassignedInfo(new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED,
                         "force allocation from previous reason " + unassigned.unassignedInfo().getReason() + ", " + unassigned.unassignedInfo().getMessage(), unassigned.unassignedInfo().getFailure()));
             }
-            it.initialize(routingNode.nodeId());
+            it.initialize(routingNode.nodeId(), unassigned.version(), allocation.clusterInfo().getShardSize(unassigned, ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE));
             break;
         }
         return new RerouteExplanation(this, decision);

core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/MoveAllocationCommand.java

@@ -178,7 +178,7 @@ public class MoveAllocationCommand implements AllocationCommand {
             if (decision.type() == Decision.Type.THROTTLE) {
                 // its being throttled, maybe have a flag to take it into account and fail? for now, just do it since the "user" wants it...
             }
-            allocation.routingNodes().relocate(shardRouting, toRoutingNode.nodeId());
+            allocation.routingNodes().relocate(shardRouting, toRoutingNode.nodeId(), allocation.clusterInfo().getShardSize(shardRouting, ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE));
         }
 
         if (!found) {

core/src/main/java/org/elasticsearch/common/logging/Loggers.java

@@ -84,6 +84,7 @@ public class Loggers {
         }
     }
 
+    @SuppressForbidden(reason = "do not know what this method does")
     public static ESLogger getLogger(String loggerName, Settings settings, String... prefixes) {
         List<String> prefixesList = newArrayList();
         if (settings.getAsBoolean("logger.logHostAddress", false)) {

core/src/main/java/org/elasticsearch/common/network/IfConfig.java

@@ -114,22 +114,22 @@ final class IfConfig {
         InetAddress address = interfaceAddress.getAddress();
         if (address instanceof Inet6Address) {
             sb.append("inet6 ");
-            sb.append(address.toString().substring(1));
+            sb.append(NetworkAddress.formatAddress(address));
             sb.append(" prefixlen:");
             sb.append(interfaceAddress.getNetworkPrefixLength());
         } else {
             sb.append("inet ");
-            sb.append(address.toString().substring(1));
+            sb.append(NetworkAddress.formatAddress(address));
             int netmask = 0xFFFFFFFF << (32 - interfaceAddress.getNetworkPrefixLength());
-            sb.append(" netmask:" + InetAddress.getByAddress(new byte[] {
+            sb.append(" netmask:" + NetworkAddress.formatAddress(InetAddress.getByAddress(new byte[] {
                     (byte)(netmask >>> 24), 
                     (byte)(netmask >>> 16 & 0xFF), 
                     (byte)(netmask >>> 8 & 0xFF), 
                     (byte)(netmask & 0xFF) 
-            }).toString().substring(1));
+            })));
             InetAddress broadcast = interfaceAddress.getBroadcast();
             if (broadcast != null) {
-                sb.append(" broadcast:" + broadcast.toString().substring(1));
+                sb.append(" broadcast:" + NetworkAddress.formatAddress(broadcast));
             }
         }
         if (address.isLoopbackAddress()) {

core/src/main/java/org/elasticsearch/common/network/NetworkAddress.java

@@ -0,0 +1,183 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.network;
+
+import com.google.common.net.InetAddresses;
+
+import org.elasticsearch.common.SuppressForbidden;
+
+import java.net.Inet6Address;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.util.Objects;
+
+/** 
+ * Utility functions for presentation of network addresses.
+ * <p>
+ * Java's address formatting is particularly bad, every address
+ * has an optional host if its resolved, so IPv4 addresses often
+ * look like this (note the confusing leading slash):
+ * <pre>
+ *    {@code /127.0.0.1}
+ * </pre>
+ * IPv6 addresses are even worse, with no IPv6 address compression,
+ * and often containing things like numeric scopeids, which are even
+ * more confusing (e.g. not going to work in any user's browser, refer
+ * to an interface on <b>another</b> machine, etc):
+ * <pre>
+ *    {@code /0:0:0:0:0:0:0:1%1}
+ * </pre>
+ * This class provides sane address formatting instead, e.g. 
+ * {@code 127.0.0.1} and {@code ::1} respectively. No methods do reverse
+ * lookups.
+ */
+public final class NetworkAddress {
+    /** No instantiation */
+    private NetworkAddress() {}
+
+    /**
+     * Formats a network address (with optional host) for display purposes.
+     * <p>
+     * If the host is already resolved (typically because, we looked up
+     * a name to do that), then we include it, otherwise it is
+     * omitted. See {@link #formatAddress(InetAddress)} if you only
+     * want the address.
+     * <p>
+     * IPv6 addresses are compressed and without scope
+     * identifiers.
+     * <p>
+     * Example output with already-resolved hostnames:
+     * <ul>
+     *   <li>IPv4: {@code localhost/127.0.0.1}</li>
+     *   <li>IPv6: {@code localhost/::1}</li>
+     * </ul>
+     * <p>
+     * Example output with just an address:
+     * <ul>
+     *   <li>IPv4: {@code 127.0.0.1}</li>
+     *   <li>IPv6: {@code ::1}</li>
+     * </ul>
+     * @param address IPv4 or IPv6 address
+     * @return formatted string
+     * @see #formatAddress(InetAddress)
+     */
+    public static String format(InetAddress address) {
+        return format(address, -1, true);
+    }
+
+    /**
+     * Formats a network address and port for display purposes.
+     * <p>
+     * If the host is already resolved (typically because, we looked up
+     * a name to do that), then we include it, otherwise it is
+     * omitted. See {@link #formatAddress(InetSocketAddress)} if you only
+     * want the address.
+     * <p>
+     * This formats the address with {@link #format(InetAddress)}
+     * and appends the port number. IPv6 addresses will be bracketed.
+     * <p>
+     * Example output with already-resolved hostnames:
+     * <ul>
+     *   <li>IPv4: {@code localhost/127.0.0.1:9300}</li>
+     *   <li>IPv6: {@code localhost/[::1]:9300}</li>
+     * </ul>
+     * <p>
+     * Example output with just an address:
+     * <ul>
+     *   <li>IPv4: {@code 127.0.0.1:9300}</li>
+     *   <li>IPv6: {@code [::1]:9300}</li>
+     * </ul>
+     * @param address IPv4 or IPv6 address with port
+     * @return formatted string
+     * @see #formatAddress(InetSocketAddress)
+     */
+    public static String format(InetSocketAddress address) {
+        return format(address.getAddress(), address.getPort(), true);
+    }
+    
+    /**
+     * Formats a network address for display purposes.
+     * <p>
+     * This formats only the address, any hostname information,
+     * if present, is ignored. IPv6 addresses are compressed 
+     * and without scope identifiers.
+     * <p>
+     * Example output with just an address:
+     * <ul>
+     *   <li>IPv4: {@code 127.0.0.1}</li>
+     *   <li>IPv6: {@code ::1}</li>
+     * </ul>
+     * @param address IPv4 or IPv6 address
+     * @return formatted string
+     */
+    public static String formatAddress(InetAddress address) {
+        return format(address, -1, false);
+    }
+    
+    /**
+     * Formats a network address and port for display purposes.
+     * <p>
+     * This formats the address with {@link #formatAddress(InetAddress)}
+     * and appends the port number. IPv6 addresses will be bracketed.
+     * Any host information, if present is ignored.
+     * <p>
+     * Example output:
+     * <ul>
+     *   <li>IPv4: {@code 127.0.0.1:9300}</li>
+     *   <li>IPv6: {@code [::1]:9300}</li>
+     * </ul>
+     * @param address IPv4 or IPv6 address with port
+     * @return formatted string
+     */
+    public static String formatAddress(InetSocketAddress address) {
+        return format(address.getAddress(), address.getPort(), false);
+    }
+    
+    // note, we don't validate port, because we only allow InetSocketAddress
+    @SuppressForbidden(reason = "we call toString to avoid a DNS lookup")
+    static String format(InetAddress address, int port, boolean includeHost) {
+        Objects.requireNonNull(address);
+        
+        StringBuilder builder = new StringBuilder();
+
+        if (includeHost) {
+            // must use toString, to avoid DNS lookup. but the format is specified in the spec
+            String toString = address.toString();
+            int separator = toString.indexOf('/');
+            if (separator > 0) {
+                // append hostname, with the slash too
+                builder.append(toString, 0, separator + 1);
+            }
+        }
+                
+        if (port != -1 && address instanceof Inet6Address) {
+            builder.append(InetAddresses.toUriString(address));
+        } else {
+            builder.append(InetAddresses.toAddrString(address));
+        }
+        
+        if (port != -1) {
+            builder.append(':');
+            builder.append(port);
+        }
+        
+        return builder.toString();
+    }
+}

core/src/main/java/org/elasticsearch/common/network/NetworkService.java

@@ -83,7 +83,6 @@ public class NetworkService extends AbstractComponent {
     public NetworkService(Settings settings) {
         super(settings);
         IfConfig.logIfNecessary();
-        InetSocketTransportAddress.setResolveAddress(settings.getAsBoolean("network.address.serialization.resolve", false));
     }
 
     /**

core/src/main/java/org/elasticsearch/common/network/NetworkUtils.java

@@ -52,6 +52,31 @@ public abstract class NetworkUtils {
      */
     @Deprecated
     static final boolean PREFER_V6 = Boolean.parseBoolean(System.getProperty("java.net.preferIPv6Addresses", "false"));
+
+    /**
+     * True if we can bind to a v6 address. Its silly, but for *binding* we have a need to know
+     * if the stack works. this can prevent scary noise on IPv4-only hosts.
+     * @deprecated transition mechanism only, do not use
+     */
+    @Deprecated
+    public static final boolean SUPPORTS_V6;
+
+    static {
+        boolean v = false;
+        try {
+            for (NetworkInterface nic : getInterfaces()) {
+                for (InetAddress address : Collections.list(nic.getInetAddresses())) {
+                    if (address instanceof Inet6Address) {
+                        v = true;
+                        break;
+                    }
+                }
+            }
+        } catch (SecurityException | SocketException misconfiguration) {
+            v = true; // be optimistic, you misconfigure, then you get noise to your screen
+        }
+        SUPPORTS_V6 = v;
+    }
     
     /** Sorts an address by preference. This way code like publishing can just pick the first one */
     static int sortKey(InetAddress address, boolean prefer_v6) {

core/src/main/java/org/elasticsearch/common/settings/Settings.java

@@ -28,7 +28,6 @@ import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 import org.elasticsearch.Version;
 import org.elasticsearch.common.Booleans;
-import org.elasticsearch.common.Classes;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.Streams;
 import org.elasticsearch.common.io.stream.StreamInput;

core/src/main/java/org/elasticsearch/common/transport/InetSocketTransportAddress.java

@@ -21,9 +21,9 @@ package org.elasticsearch.common.transport;
 
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.network.NetworkAddress;
 
 import java.io.IOException;
-import java.net.Inet6Address;
 import java.net.InetAddress;
 import java.net.InetSocketAddress;
 
@@ -32,52 +32,34 @@ import java.net.InetSocketAddress;
  */
 public final class InetSocketTransportAddress implements TransportAddress {
 
-    private static boolean resolveAddress = false;
-
-    public static void setResolveAddress(boolean resolveAddress) {
-        InetSocketTransportAddress.resolveAddress = resolveAddress;
-    }
-
-    public static boolean getResolveAddress() {
-        return resolveAddress;
-    }
-
     public static final InetSocketTransportAddress PROTO = new InetSocketTransportAddress();
 
     private final InetSocketAddress address;
 
     public InetSocketTransportAddress(StreamInput in) throws IOException {
-        if (in.readByte() == 0) {
-            int len = in.readByte();
-            byte[] a = new byte[len]; // 4 bytes (IPv4) or 16 bytes (IPv6)
-            in.readFully(a);
-            InetAddress inetAddress;
-            if (len == 16) {
-                int scope_id = in.readInt();
-                inetAddress = Inet6Address.getByAddress(null, a, scope_id);
-            } else {
-                inetAddress = InetAddress.getByAddress(a);
-            }
-            int port = in.readInt();
-            this.address = new InetSocketAddress(inetAddress, port);
-        } else {
-            this.address = new InetSocketAddress(in.readString(), in.readInt());
-        }
+        final int len = in.readByte();
+        final byte[] a = new byte[len]; // 4 bytes (IPv4) or 16 bytes (IPv6)
+        in.readFully(a);
+        InetAddress inetAddress = InetAddress.getByAddress(a);
+        int port = in.readInt();
+        this.address = new InetSocketAddress(inetAddress, port);
     }
 
     private InetSocketTransportAddress() {
         address = null;
     }
 
-    public InetSocketTransportAddress(String hostname, int port) {
-        this(new InetSocketAddress(hostname, port));
-    }
-
     public InetSocketTransportAddress(InetAddress address, int port) {
         this(new InetSocketAddress(address, port));
     }
 
     public InetSocketTransportAddress(InetSocketAddress address) {
+        if (address == null) {
+            throw new IllegalArgumentException("InetSocketAddress must not be null");
+        }
+        if (address.getAddress() == null) {
+            throw new IllegalArgumentException("Address must be resolved but wasn't - InetSocketAddress#getAddress() returned null");
+        }
         this.address = address;
     }
 
@@ -94,16 +76,12 @@ public final class InetSocketTransportAddress implements TransportAddress {
 
     @Override
     public String getHost() {
-        if (resolveAddress) {
-            return address.getHostName();
-        } else {
-            return getAddress();
-        }
+       return getAddress(); // just delegate no resolving
     }
 
     @Override
     public String getAddress() {
-        return address.getAddress().getHostAddress();
+        return NetworkAddress.formatAddress(address.getAddress());
     }
 
     @Override
@@ -122,20 +100,16 @@ public final class InetSocketTransportAddress implements TransportAddress {
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        if (!resolveAddress && address.getAddress() != null) {
-            out.writeByte((byte) 0);
-            byte[] bytes = address().getAddress().getAddress();  // 4 bytes (IPv4) or 16 bytes (IPv6)
-            out.writeByte((byte) bytes.length); // 1 byte
-            out.write(bytes, 0, bytes.length);
-            if (address().getAddress() instanceof Inet6Address)
-                out.writeInt(((Inet6Address) address.getAddress()).getScopeId());
-        } else {
-            out.writeByte((byte) 1);
-            out.writeString(address.getHostName());
-        }
+        byte[] bytes = address().getAddress().getAddress();  // 4 bytes (IPv4) or 16 bytes (IPv6)
+        out.writeByte((byte) bytes.length); // 1 byte
+        out.write(bytes, 0, bytes.length);
+        // don't serialize scope ids over the network!!!!
+        // these only make sense with respect to the local machine, and will only formulate
+        // the address incorrectly remotely.
         out.writeInt(address.getPort());
     }
 
+
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;
@@ -151,6 +125,6 @@ public final class InetSocketTransportAddress implements TransportAddress {
 
     @Override
     public String toString() {
-        return "inet[" + address + "]";
+        return NetworkAddress.format(address);
     }
 }

core/src/main/java/org/elasticsearch/common/util/ExtensionPoint.java

@@ -187,7 +187,7 @@ public abstract class ExtensionPoint {
         protected final void bindExtensions(Binder binder) {
             Multibinder<T> allocationMultibinder = Multibinder.newSetBinder(binder, extensionClass);
             for (Class<? extends T> clazz : extensions) {
-                allocationMultibinder.addBinding().to(clazz);
+                allocationMultibinder.addBinding().to(clazz).asEagerSingleton();
             }
         }
     }

core/src/main/java/org/elasticsearch/discovery/DiscoveryModule.java

@@ -19,18 +19,20 @@
 
 package org.elasticsearch.discovery;
 
-import com.google.common.collect.Lists;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.inject.AbstractModule;
 import org.elasticsearch.common.inject.multibindings.Multibinder;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.ExtensionPoint;
 import org.elasticsearch.discovery.local.LocalDiscovery;
 import org.elasticsearch.discovery.zen.ZenDiscovery;
 import org.elasticsearch.discovery.zen.elect.ElectMasterService;
+import org.elasticsearch.discovery.zen.ping.ZenPing;
 import org.elasticsearch.discovery.zen.ping.ZenPingService;
 import org.elasticsearch.discovery.zen.ping.unicast.UnicastHostsProvider;
+import org.elasticsearch.discovery.zen.ping.unicast.UnicastZenPing;
 
+import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -44,7 +46,8 @@ public class DiscoveryModule extends AbstractModule {
     public static final String ZEN_MASTER_SERVICE_TYPE_KEY = "discovery.zen.masterservice.type";
 
     private final Settings settings;
-    private final List<Class<? extends UnicastHostsProvider>> unicastHostProviders = Lists.newArrayList();
+    private final List<Class<? extends UnicastHostsProvider>> unicastHostProviders = new ArrayList<>();
+    private final ExtensionPoint.ClassSet<ZenPing> zenPings = new ExtensionPoint.ClassSet<>("zen_ping", ZenPing.class);
     private final Map<String, Class<? extends Discovery>> discoveryTypes = new HashMap<>();
     private final Map<String, Class<? extends ElectMasterService>> masterServiceType = new HashMap<>();
 
@@ -53,6 +56,8 @@ public class DiscoveryModule extends AbstractModule {
         addDiscoveryType("local", LocalDiscovery.class);
         addDiscoveryType("zen", ZenDiscovery.class);
         addElectMasterService("zen", ElectMasterService.class);
+        // always add the unicast hosts, or things get angry!
+        addZenPing(UnicastZenPing.class);
     }
 
     /**
@@ -82,6 +87,10 @@ public class DiscoveryModule extends AbstractModule {
         this.masterServiceType.put(type, masterService);
     }
 
+    public void addZenPing(Class<? extends ZenPing> clazz) {
+        zenPings.registerExtension(clazz);
+    }
+
     @Override
     protected void configure() {
         String defaultType = DiscoveryNode.localNode(settings) ? "local" : "zen";
@@ -107,6 +116,7 @@ public class DiscoveryModule extends AbstractModule {
             for (Class<? extends UnicastHostsProvider> unicastHostProvider : unicastHostProviders) {
                 unicastHostsProviderMultibinder.addBinding().to(unicastHostProvider);
             }
+            zenPings.bind(binder());
         }
         bind(Discovery.class).to(discoveryClass).asEagerSingleton();
         bind(DiscoveryService.class).asEagerSingleton();

core/src/main/java/org/elasticsearch/discovery/zen/ping/ZenPingService.java

@@ -19,50 +19,31 @@
 
 package org.elasticsearch.discovery.zen.ping;
 
-import com.google.common.collect.ImmutableList;
-import org.elasticsearch.Version;
-import org.elasticsearch.cluster.ClusterName;
-import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
 import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
-import org.elasticsearch.discovery.zen.elect.ElectMasterService;
-import org.elasticsearch.discovery.zen.ping.multicast.MulticastZenPing;
-import org.elasticsearch.discovery.zen.ping.unicast.UnicastHostsProvider;
-import org.elasticsearch.discovery.zen.ping.unicast.UnicastZenPing;
-import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.transport.TransportService;
 
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
 import java.util.Set;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
 
-/**
- *
- */
 public class ZenPingService extends AbstractLifecycleComponent<ZenPing> implements ZenPing {
 
-    private volatile ImmutableList<? extends ZenPing> zenPings = ImmutableList.of();
+    private List<ZenPing> zenPings = Collections.emptyList();
 
     @Inject
-    public ZenPingService(Settings settings, ThreadPool threadPool, TransportService transportService, ClusterName clusterName, NetworkService networkService,
-                          Version version, ElectMasterService electMasterService, @Nullable Set<UnicastHostsProvider> unicastHostsProviders) {
+    public ZenPingService(Settings settings, Set<ZenPing> zenPings) {
         super(settings);
-        ImmutableList.Builder<ZenPing> zenPingsBuilder = ImmutableList.builder();
-        if (this.settings.getAsBoolean("discovery.zen.ping.multicast.enabled", true)) {
-            zenPingsBuilder.add(new MulticastZenPing(settings, threadPool, transportService, clusterName, networkService, version));
-        }
-        // always add the unicast hosts, so it will be able to receive unicast requests even when working in multicast
-        zenPingsBuilder.add(new UnicastZenPing(settings, threadPool, transportService, clusterName, version, electMasterService, unicastHostsProviders));
-
-        this.zenPings = zenPingsBuilder.build();
+        this.zenPings = Collections.unmodifiableList(new ArrayList<>(zenPings));
     }
 
-    public ImmutableList<? extends ZenPing> zenPings() {
+    public List<ZenPing> zenPings() {
         return this.zenPings;
     }
 
@@ -118,7 +99,7 @@ public class ZenPingService extends AbstractLifecycleComponent<ZenPing> implemen
 
     @Override
     public void ping(PingListener listener, TimeValue timeout) {
-        ImmutableList<? extends ZenPing> zenPings = this.zenPings;
+        List<? extends ZenPing> zenPings = this.zenPings;
         CompoundPingListener compoundPingListener = new CompoundPingListener(listener, zenPings);
         for (ZenPing zenPing : zenPings) {
             try {
@@ -138,7 +119,7 @@ public class ZenPingService extends AbstractLifecycleComponent<ZenPing> implemen
 
         private PingCollection responses = new PingCollection();
 
-        private CompoundPingListener(PingListener listener, ImmutableList<? extends ZenPing> zenPings) {
+        private CompoundPingListener(PingListener listener, List<? extends ZenPing> zenPings) {
             this.listener = listener;
             this.counter = new AtomicInteger(zenPings.size());
         }

core/src/main/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPing.java

@@ -29,6 +29,7 @@ import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
+import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Settings;
@@ -64,7 +65,10 @@ public class UnicastZenPing extends AbstractLifecycleComponent<ZenPing> implemen
 
     public static final String ACTION_NAME = "internal:discovery/zen/unicast";
 
-    public static final int LIMIT_PORTS_COUNT = 1;
+    // these limits are per-address
+    public static final int LIMIT_FOREIGN_PORTS_COUNT = 1;
+    public static final int LIMIT_LOCAL_PORTS_COUNT = 5;
+
 
     private final ThreadPool threadPool;
     private final TransportService transportService;
@@ -96,6 +100,7 @@ public class UnicastZenPing extends AbstractLifecycleComponent<ZenPing> implemen
 
     private volatile boolean closed = false;
 
+    @Inject
     public UnicastZenPing(Settings settings, ThreadPool threadPool, TransportService transportService, ClusterName clusterName,
                           Version version, ElectMasterService electMasterService, @Nullable Set<UnicastHostsProvider> unicastHostsProviders) {
         super(settings);
@@ -117,15 +122,24 @@ public class UnicastZenPing extends AbstractLifecycleComponent<ZenPing> implemen
             hostArr[i] = hostArr[i].trim();
         }
         List<String> hosts = Lists.newArrayList(hostArr);
+        final int limitPortCounts;
+        if (hosts.isEmpty()) {
+            // if unicast hosts are not specified, fill with simple defaults on the local machine
+            limitPortCounts = LIMIT_LOCAL_PORTS_COUNT;
+            hosts.addAll(transportService.getLocalAddresses());
+        } else {
+            // we only limit to 1 addresses, makes no sense to ping 100 ports
+            limitPortCounts = LIMIT_FOREIGN_PORTS_COUNT;
+        }
+
         logger.debug("using initial hosts {}, with concurrent_connects [{}]", hosts, concurrentConnects);
 
         List<DiscoveryNode> configuredTargetNodes = Lists.newArrayList();
         for (String host : hosts) {
             try {
-                TransportAddress[] addresses = transportService.addressesFromString(host);
-                // we only limit to 1 addresses, makes no sense to ping 100 ports
-                for (int i = 0; (i < addresses.length && i < LIMIT_PORTS_COUNT); i++) {
-                    configuredTargetNodes.add(new DiscoveryNode(UNICAST_NODE_PREFIX + unicastNodeIdGenerator.incrementAndGet() + "#", addresses[i], version.minimumCompatibilityVersion()));
+                TransportAddress[] addresses = transportService.addressesFromString(host, limitPortCounts);
+                for (TransportAddress address : addresses) {
+                    configuredTargetNodes.add(new DiscoveryNode(UNICAST_NODE_PREFIX + unicastNodeIdGenerator.incrementAndGet() + "#", address, version.minimumCompatibilityVersion()));
                 }
             } catch (Exception e) {
                 throw new IllegalArgumentException("Failed to resolve address for [" + host + "]", e);

core/src/main/java/org/elasticsearch/gateway/PrimaryShardAllocator.java

@@ -94,12 +94,12 @@ public abstract class PrimaryShardAllocator extends AbstractComponent {
                 DiscoveryNode node = nodesToAllocate.yesNodes.get(0);
                 logger.debug("[{}][{}]: allocating [{}] to [{}] on primary allocation", shard.index(), shard.id(), shard, node);
                 changed = true;
-                unassignedIterator.initialize(node.id(), nodesAndVersions.highestVersion);
+                unassignedIterator.initialize(node.id(), nodesAndVersions.highestVersion, ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE);
             } else if (nodesToAllocate.throttleNodes.isEmpty() == true && nodesToAllocate.noNodes.isEmpty() == false) {
                 DiscoveryNode node = nodesToAllocate.noNodes.get(0);
                 logger.debug("[{}][{}]: forcing allocating [{}] to [{}] on primary allocation", shard.index(), shard.id(), shard, node);
                 changed = true;
-                unassignedIterator.initialize(node.id(), nodesAndVersions.highestVersion);
+                unassignedIterator.initialize(node.id(), nodesAndVersions.highestVersion, ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE);
             } else {
                 // we are throttling this, but we have enough to allocate to this node, ignore it for now
                 logger.debug("[{}][{}]: throttling allocation [{}] to [{}] on primary allocation", shard.index(), shard.id(), shard, nodesToAllocate.throttleNodes);

core/src/main/java/org/elasticsearch/gateway/ReplicaShardAllocator.java

@@ -169,7 +169,7 @@ public abstract class ReplicaShardAllocator extends AbstractComponent {
                     logger.debug("[{}][{}]: allocating [{}] to [{}] in order to reuse its unallocated persistent store", shard.index(), shard.id(), shard, nodeWithHighestMatch.node());
                     // we found a match
                     changed = true;
-                    unassignedIterator.initialize(nodeWithHighestMatch.nodeId());
+                    unassignedIterator.initialize(nodeWithHighestMatch.nodeId(), shard.version(), allocation.clusterInfo().getShardSize(shard, ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE));
                 }
             } else if (matchingNodes.hasAnyData() == false) {
                 // if we didn't manage to find *any* data (regardless of matching sizes), check if the allocation

core/src/main/java/org/elasticsearch/http/netty/NettyHttpServerTransport.java

@@ -24,6 +24,7 @@ import org.elasticsearch.common.component.AbstractLifecycleComponent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.netty.NettyUtils;
 import org.elasticsearch.common.netty.OpenChannelsHandler;
+import org.elasticsearch.common.network.NetworkAddress;
 import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.network.NetworkUtils;
 import org.elasticsearch.common.settings.Settings;
@@ -274,7 +275,7 @@ public class NettyHttpServerTransport extends AbstractLifecycleComponent<HttpSer
     private void bindAddress(final InetAddress hostAddress) {
         PortsRange portsRange = new PortsRange(port);
         final AtomicReference<Exception> lastException = new AtomicReference<>();
-        final AtomicReference<SocketAddress> boundSocket = new AtomicReference<>();
+        final AtomicReference<InetSocketAddress> boundSocket = new AtomicReference<>();
         boolean success = portsRange.iterate(new PortsRange.PortCallback() {
             @Override
             public boolean onPortNumber(int portNumber) {
@@ -282,7 +283,7 @@ public class NettyHttpServerTransport extends AbstractLifecycleComponent<HttpSer
                     synchronized (serverChannels) {
                         Channel channel = serverBootstrap.bind(new InetSocketAddress(hostAddress, portNumber));
                         serverChannels.add(channel);
-                        boundSocket.set(channel.getLocalAddress());
+                        boundSocket.set((InetSocketAddress) channel.getLocalAddress());
                     }
                 } catch (Exception e) {
                     lastException.set(e);
@@ -294,7 +295,7 @@ public class NettyHttpServerTransport extends AbstractLifecycleComponent<HttpSer
         if (!success) {
             throw new BindHttpException("Failed to bind to [" + port + "]", lastException.get());
         }
-        logger.info("Bound http to address [{}]", boundSocket.get());
+        logger.info("Bound http to address {{}}", NetworkAddress.format(boundSocket.get()));
     }
 
     @Override

core/src/main/java/org/elasticsearch/index/IndexService.java

@@ -26,6 +26,7 @@ import com.google.common.collect.Iterators;
 import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.Tuple;
@@ -270,7 +271,8 @@ public class IndexService extends AbstractIndexComponent implements IndexCompone
         }
     }
 
-    public synchronized IndexShard createShard(int sShardId, boolean primary) {
+    public synchronized IndexShard createShard(int sShardId, ShardRouting routing) {
+        final boolean primary = routing.primary();
         /*
          * TODO: we execute this in parallel but it's a synced method. Yet, we might
          * be able to serialize the execution via the cluster state in the future. for now we just
@@ -299,7 +301,7 @@ public class IndexService extends AbstractIndexComponent implements IndexCompone
                 }
             }
             if (path == null) {
-                path = ShardPath.selectNewPathForShard(nodeEnv, shardId, indexSettings, getAvgShardSizeInBytes(), this);
+                path = ShardPath.selectNewPathForShard(nodeEnv, shardId, indexSettings, routing.getExpectedShardSize() == ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE ? getAvgShardSizeInBytes() : routing.getExpectedShardSize(), this);
                 logger.debug("{} creating using a new path [{}]", shardId, path);
             } else {
                 logger.debug("{} creating using an existing path [{}]", shardId, path);

core/src/main/java/org/elasticsearch/index/shard/IndexShard.java

@@ -21,8 +21,15 @@ package org.elasticsearch.index.shard;
 
 import com.google.common.base.Charsets;
 import com.google.common.base.Preconditions;
+
 import org.apache.lucene.codecs.PostingsFormat;
 import org.apache.lucene.index.CheckIndex;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.DisjunctionMaxQuery;
+import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.MatchNoDocsQuery;
+import org.apache.lucene.search.Query;
 import org.apache.lucene.search.QueryCachingPolicy;
 import org.apache.lucene.search.UsageTrackingQueryCachingPolicy;
 import org.apache.lucene.store.AlreadyClosedException;
@@ -162,8 +169,8 @@ public class IndexShard extends AbstractIndexShardComponent {
 
     private TimeValue refreshInterval;
 
-    private volatile ScheduledFuture refreshScheduledFuture;
-    private volatile ScheduledFuture mergeScheduleFuture;
+    private volatile ScheduledFuture<?> refreshScheduledFuture;
+    private volatile ScheduledFuture<?> mergeScheduleFuture;
     protected volatile ShardRouting shardRouting;
     protected volatile IndexShardState state;
     protected final AtomicReference<Engine> currentEngineReference = new AtomicReference<>();
@@ -252,7 +259,42 @@ public class IndexShard extends AbstractIndexShardComponent {
         if (indexSettings.getAsBoolean(IndexCacheModule.QUERY_CACHE_EVERYTHING, false)) {
             cachingPolicy = QueryCachingPolicy.ALWAYS_CACHE;
         } else {
-            cachingPolicy = new UsageTrackingQueryCachingPolicy();
+            assert Version.CURRENT.luceneVersion == org.apache.lucene.util.Version.LUCENE_5_2_1;
+            // TODO: remove this hack in Lucene 5.4, use UsageTrackingQueryCachingPolicy directly
+            // See https://issues.apache.org/jira/browse/LUCENE-6748
+            // cachingPolicy = new UsageTrackingQueryCachingPolicy();
+
+            final QueryCachingPolicy wrapped = new UsageTrackingQueryCachingPolicy();
+            cachingPolicy = new QueryCachingPolicy() {
+
+                @Override
+                public boolean shouldCache(Query query, LeafReaderContext context) throws IOException {
+                    if (query instanceof MatchAllDocsQuery
+                            // MatchNoDocsQuery currently rewrites to a BooleanQuery,
+                            // but who knows, it might get its own Weight one day
+                            || query instanceof MatchNoDocsQuery) {
+                        return false;
+                    }
+                    if (query instanceof BooleanQuery) {
+                        BooleanQuery bq = (BooleanQuery) query;
+                        if (bq.clauses().isEmpty()) {
+                            return false;
+                        }
+                    }
+                    if (query instanceof DisjunctionMaxQuery) {
+                        DisjunctionMaxQuery dmq = (DisjunctionMaxQuery) query;
+                        if (dmq.getDisjuncts().isEmpty()) {
+                            return false;
+                        }
+                    }
+                    return wrapped.shouldCache(query, context);
+                }
+
+                @Override
+                public void onUse(Query query) {
+                    wrapped.onUse(query);
+                }
+            };
         }
         this.engineConfig = newEngineConfig(translogConfig, cachingPolicy);
         this.indexShardOperationCounter = new IndexShardOperationCounter(logger, shardId);

core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java

@@ -638,7 +638,7 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent<Indic
                 if (logger.isDebugEnabled()) {
                     logger.debug("[{}][{}] creating shard", shardRouting.index(), shardId);
                 }
-                IndexShard indexShard = indexService.createShard(shardId, shardRouting.primary());
+                IndexShard indexShard = indexService.createShard(shardId, shardRouting);
                 indexShard.updateRoutingEntry(shardRouting, state.blocks().disableStatePersistence() == false);
                 indexShard.addFailedEngineListener(failedEngineHandler);
             } catch (IndexShardAlreadyExistsException e) {

core/src/main/java/org/elasticsearch/plugins/PluginManager.java

@@ -84,6 +84,7 @@ public class PluginManager {
                     "cloud-azure",
                     "cloud-gce",
                     "delete-by-query",
+                    "discovery-multicast",
                     "lang-javascript",
                     "lang-python",
                     "mapper-murmur3",

core/src/main/java/org/elasticsearch/search/SearchService.java

@@ -263,6 +263,7 @@ public class SearchService extends AbstractLifecycleComponent<SearchService> {
         }
     }
 
+    @Deprecated // remove in 3.0
     public QuerySearchResult executeScan(ShardSearchRequest request) {
         final SearchContext context = createAndPutContext(request);
         final int originalSize = context.size();

core/src/main/java/org/elasticsearch/search/action/SearchServiceTransportAction.java

@@ -418,6 +418,7 @@ public class SearchServiceTransportAction extends AbstractComponent {
         }
     }
 
+    @Deprecated // remove in 3.0
     class SearchScanTransportHandler implements TransportRequestHandler<ShardSearchTransportRequest> {
         @Override
         public void messageReceived(ShardSearchTransportRequest request, TransportChannel channel) throws Exception {

core/src/main/java/org/elasticsearch/transport/Transport.java

@@ -25,6 +25,7 @@ import org.elasticsearch.common.transport.BoundTransportAddress;
 import org.elasticsearch.common.transport.TransportAddress;
 
 import java.io.IOException;
+import java.util.List;
 import java.util.Map;
 
 /**
@@ -32,6 +33,7 @@ import java.util.Map;
  */
 public interface Transport extends LifecycleComponent<Transport> {
 
+
     public static class TransportSettings {
         public static final String TRANSPORT_TCP_COMPRESS = "transport.tcp.compress";
     }
@@ -52,7 +54,7 @@ public interface Transport extends LifecycleComponent<Transport> {
     /**
      * Returns an address from its string representation.
      */
-    TransportAddress[] addressesFromString(String address) throws Exception;
+    TransportAddress[] addressesFromString(String address, int perAddressLimit) throws Exception;
 
     /**
      * Is the address type supported.
@@ -89,4 +91,6 @@ public interface Transport extends LifecycleComponent<Transport> {
      * Returns count of currently open connections
      */
     long serverOpen();
+
+    List<String> getLocalAddresses();
 }

core/src/main/java/org/elasticsearch/transport/TransportModule.java

@@ -92,7 +92,6 @@ public class TransportModule extends AbstractModule {
         }
 
         bind(NamedWriteableRegistry.class).asEagerSingleton();
-
         if (configuredTransport != null) {
             logger.info("Using [{}] as transport, overridden by [{}]", configuredTransport.getName(), configuredTransportSource);
             bind(Transport.class).to(configuredTransport).asEagerSingleton();

core/src/main/java/org/elasticsearch/transport/TransportService.java

@@ -40,10 +40,7 @@ import org.elasticsearch.node.settings.NodeSettingsService;
 import org.elasticsearch.threadpool.ThreadPool;
 
 import java.io.IOException;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.LinkedHashMap;
-import java.util.Map;
+import java.util.*;
 import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.concurrent.ScheduledFuture;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -221,6 +218,10 @@ public class TransportService extends AbstractLifecycleComponent<TransportServic
         return transport.boundAddress();
     }
 
+    public List<String> getLocalAddresses() {
+        return transport.getLocalAddresses();
+    }
+
     public boolean nodeConnected(DiscoveryNode node) {
         return node.equals(localNode) || transport.nodeConnected(node);
     }
@@ -383,8 +384,8 @@ public class TransportService extends AbstractLifecycleComponent<TransportServic
         return requestIds.getAndIncrement();
     }
 
-    public TransportAddress[] addressesFromString(String address) throws Exception {
-        return transport.addressesFromString(address);
+    public TransportAddress[] addressesFromString(String address, int perAddressLimit) throws Exception {
+        return transport.addressesFromString(address, perAddressLimit);
     }
 
     /**

core/src/main/java/org/elasticsearch/transport/local/LocalTransport.java

@@ -41,8 +41,7 @@ import org.elasticsearch.transport.*;
 import org.elasticsearch.transport.support.TransportStatus;
 
 import java.io.IOException;
-import java.util.Collections;
-import java.util.Map;
+import java.util.*;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.ThreadFactory;
 import java.util.concurrent.ThreadPoolExecutor;
@@ -57,14 +56,13 @@ import static org.elasticsearch.common.util.concurrent.ConcurrentCollections.new
 public class LocalTransport extends AbstractLifecycleComponent<Transport> implements Transport {
 
     public static final String LOCAL_TRANSPORT_THREAD_NAME_PREFIX = "local_transport";
-
     private final ThreadPool threadPool;
     private final ThreadPoolExecutor workers;
     private final Version version;
     private volatile TransportServiceAdapter transportServiceAdapter;
     private volatile BoundTransportAddress boundAddress;
     private volatile LocalTransportAddress localAddress;
-    private final static ConcurrentMap<TransportAddress, LocalTransport> transports = newConcurrentMap();
+    private final static ConcurrentMap<LocalTransportAddress, LocalTransport> transports = newConcurrentMap();
     private static final AtomicLong transportAddressIdGenerator = new AtomicLong();
     private final ConcurrentMap<DiscoveryNode, LocalTransport> connectedNodes = newConcurrentMap();
     private final NamedWriteableRegistry namedWriteableRegistry;
@@ -78,7 +76,6 @@ public class LocalTransport extends AbstractLifecycleComponent<Transport> implem
         super(settings);
         this.threadPool = threadPool;
         this.version = version;
-
         int workerCount = this.settings.getAsInt(TRANSPORT_LOCAL_WORKERS, EsExecutors.boundedNumberOfProcessors(settings));
         int queueSize = this.settings.getAsInt(TRANSPORT_LOCAL_QUEUE, -1);
         logger.debug("creating [{}] workers, queue_size [{}]", workerCount, queueSize);
@@ -88,7 +85,7 @@ public class LocalTransport extends AbstractLifecycleComponent<Transport> implem
     }
 
     @Override
-    public TransportAddress[] addressesFromString(String address) {
+    public TransportAddress[] addressesFromString(String address, int perAddressLimit) {
         return new TransportAddress[]{new LocalTransportAddress(address)};
     }
 
@@ -359,4 +356,9 @@ public class LocalTransport extends AbstractLifecycleComponent<Transport> implem
             logger.error("failed to handle exception response [{}]", t, handler);
         }
     }
+
+    @Override
+    public List<String> getLocalAddresses() {
+        return Collections.singletonList("0.0.0.0");
+    }
 }

core/src/main/java/org/elasticsearch/transport/netty/NettyTransport.java

@@ -22,8 +22,8 @@ package org.elasticsearch.transport.netty;
 import com.google.common.base.Charsets;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
+
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -42,6 +42,7 @@ import org.elasticsearch.common.metrics.CounterMetric;
 import org.elasticsearch.common.netty.NettyUtils;
 import org.elasticsearch.common.netty.OpenChannelsHandler;
 import org.elasticsearch.common.netty.ReleaseChannelFutureListener;
+import org.elasticsearch.common.network.NetworkAddress;
 import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.network.NetworkUtils;
 import org.elasticsearch.common.settings.Settings;
@@ -75,6 +76,7 @@ import java.io.IOException;
 import java.net.InetAddress;
 import java.net.InetSocketAddress;
 import java.net.SocketAddress;
+import java.net.UnknownHostException;
 import java.nio.channels.CancelledKeyException;
 import java.util.*;
 import java.util.concurrent.*;
@@ -82,6 +84,8 @@ import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.concurrent.locks.ReadWriteLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 import static org.elasticsearch.common.network.NetworkService.TcpSettings.*;
 import static org.elasticsearch.common.settings.Settings.settingsBuilder;
@@ -405,7 +409,11 @@ public class NettyTransport extends AbstractLifecycleComponent<Transport> implem
             throw new BindTransportException("Failed to resolve host [" + bindHost + "]", e);
         }
         if (logger.isDebugEnabled()) {
-            logger.debug("binding server bootstrap to: {}", hostAddresses);
+            String[] addresses = new String[hostAddresses.length];
+            for (int i = 0; i < hostAddresses.length; i++) {
+                addresses[i] = NetworkAddress.format(hostAddresses[i]);
+            }
+            logger.debug("binding server bootstrap to: {}", addresses);
         }
         for (InetAddress hostAddress : hostAddresses) {
             bindServerBootstrap(name, hostAddress, settings);
@@ -417,7 +425,7 @@ public class NettyTransport extends AbstractLifecycleComponent<Transport> implem
         String port = settings.get("port");
         PortsRange portsRange = new PortsRange(port);
         final AtomicReference<Exception> lastException = new AtomicReference<>();
-        final AtomicReference<SocketAddress> boundSocket = new AtomicReference<>();
+        final AtomicReference<InetSocketAddress> boundSocket = new AtomicReference<>();
         boolean success = portsRange.iterate(new PortsRange.PortCallback() {
             @Override
             public boolean onPortNumber(int portNumber) {
@@ -430,7 +438,7 @@ public class NettyTransport extends AbstractLifecycleComponent<Transport> implem
                             serverChannels.put(name, list);
                         }
                         list.add(channel);
-                        boundSocket.set(channel.getLocalAddress());
+                        boundSocket.set((InetSocketAddress)channel.getLocalAddress());
                     }
                 } catch (Exception e) {
                     lastException.set(e);
@@ -444,7 +452,7 @@ public class NettyTransport extends AbstractLifecycleComponent<Transport> implem
         }
 
         if (!DEFAULT_PROFILE.equals(name)) {
-            InetSocketAddress boundAddress = (InetSocketAddress) boundSocket.get();
+            InetSocketAddress boundAddress = boundSocket.get();
             int publishPort = settings.getAsInt("publish_port", boundAddress.getPort());
             String publishHost = settings.get("publish_host", boundAddress.getHostString());
             InetSocketAddress publishAddress = createPublishAddress(publishHost, publishPort);
@@ -452,7 +460,7 @@ public class NettyTransport extends AbstractLifecycleComponent<Transport> implem
             profileBoundAddresses.putIfAbsent(name, new BoundTransportAddress(new InetSocketTransportAddress(boundAddress), new InetSocketTransportAddress(publishAddress)));
         }
 
-        logger.info("Bound profile [{}] to address [{}]", name, boundSocket.get());
+        logger.info("Bound profile [{}] to address {{}}", name, NetworkAddress.format(boundSocket.get()));
     }
 
     private void createServerBootstrap(String name, Settings settings) {
@@ -582,35 +590,65 @@ public class NettyTransport extends AbstractLifecycleComponent<Transport> implem
     }
 
     @Override
-    public TransportAddress[] addressesFromString(String address) throws Exception {
-        int index = address.indexOf('[');
-        if (index != -1) {
-            String host = address.substring(0, index);
-            Set<String> ports = Strings.commaDelimitedListToSet(address.substring(index + 1, address.indexOf(']')));
-            List<TransportAddress> addresses = Lists.newArrayList();
-            for (String port : ports) {
-                int[] iPorts = new PortsRange(port).ports();
-                for (int iPort : iPorts) {
-                    addresses.add(new InetSocketTransportAddress(host, iPort));
-                }
-            }
-            return addresses.toArray(new TransportAddress[addresses.size()]);
+    public TransportAddress[] addressesFromString(String address, int perAddressLimit) throws Exception {
+        return parse(address, settings.get("transport.profiles.default.port", 
+                              settings.get("transport.netty.port", 
+                              settings.get("transport.tcp.port", 
+                              DEFAULT_PORT_RANGE))), perAddressLimit);
+    }
+    
+    // this code is a take on guava's HostAndPort, like a HostAndPortRange
+    
+    // pattern for validating ipv6 bracked addresses. 
+    // not perfect, but PortsRange should take care of any port range validation, not a regex
+    private static final Pattern BRACKET_PATTERN = Pattern.compile("^\\[(.*:.*)\\](?::([\\d\\-]*))?$");
+
+    /** parse a hostname+port range spec into its equivalent addresses */
+    static TransportAddress[] parse(String hostPortString, String defaultPortRange, int perAddressLimit) throws UnknownHostException {
+        Objects.requireNonNull(hostPortString);
+        String host;
+        String portString = null;
+
+        if (hostPortString.startsWith("[")) {
+          // Parse a bracketed host, typically an IPv6 literal.
+          Matcher matcher = BRACKET_PATTERN.matcher(hostPortString);
+          if (!matcher.matches()) {
+              throw new IllegalArgumentException("Invalid bracketed host/port range: " + hostPortString);
+          }
+          host = matcher.group(1);
+          portString = matcher.group(2);  // could be null
         } else {
-            index = address.lastIndexOf(':');
-            if (index == -1) {
-                List<TransportAddress> addresses = Lists.newArrayList();
-                String defaultPort = settings.get("transport.profiles.default.port", settings.get("transport.netty.port", this.settings.get("transport.tcp.port", DEFAULT_PORT_RANGE)));
-                int[] iPorts = new PortsRange(defaultPort).ports();
-                for (int iPort : iPorts) {
-                    addresses.add(new InetSocketTransportAddress(address, iPort));
-                }
-                return addresses.toArray(new TransportAddress[addresses.size()]);
-            } else {
-                String host = address.substring(0, index);
-                int port = Integer.parseInt(address.substring(index + 1));
-                return new TransportAddress[]{new InetSocketTransportAddress(host, port)};
+          int colonPos = hostPortString.indexOf(':');
+          if (colonPos >= 0 && hostPortString.indexOf(':', colonPos + 1) == -1) {
+            // Exactly 1 colon.  Split into host:port.
+            host = hostPortString.substring(0, colonPos);
+            portString = hostPortString.substring(colonPos + 1);
+          } else {
+            // 0 or 2+ colons.  Bare hostname or IPv6 literal.
+            host = hostPortString;
+            // 2+ colons and not bracketed: exception
+            if (colonPos >= 0) {
+                throw new IllegalArgumentException("IPv6 addresses must be bracketed: " + hostPortString);
+            }
+          }
+        }
+        
+        // if port isn't specified, fill with the default
+        if (portString == null || portString.isEmpty()) {
+            portString = defaultPortRange;
+        }
+        
+        // generate address for each port in the range
+        Set<InetAddress> addresses = new HashSet<>(Arrays.asList(InetAddress.getAllByName(host)));
+        List<TransportAddress> transportAddresses = new ArrayList<>();
+        int[] ports = new PortsRange(portString).ports();
+        int limit = Math.min(ports.length, perAddressLimit);
+        for (int i = 0; i < limit; i++) {
+            for (InetAddress address : addresses) {
+                transportAddresses.add(new InetSocketTransportAddress(address, ports[i]));
             }
         }
+        return transportAddresses.toArray(new TransportAddress[transportAddresses.size()]);
     }
 
     @Override
@@ -673,6 +711,17 @@ public class NettyTransport extends AbstractLifecycleComponent<Transport> implem
         return channels == null ? 0 : channels.numberOfOpenChannels();
     }
 
+    @Override
+    public List<String> getLocalAddresses() {
+        List<String> local = new ArrayList<>();
+        local.add("127.0.0.1");
+        // check if v6 is supported, if so, v4 will also work via mapped addresses.
+        if (NetworkUtils.SUPPORTS_V6) {
+            local.add("[::1]"); // may get ports appended!
+        }
+        return local;
+    }
+
     @Override
     public void sendRequest(final DiscoveryNode node, final long requestId, final String action, final TransportRequest request, TransportRequestOptions options) throws IOException, TransportException {
 

core/src/main/resources/org/elasticsearch/plugins/plugin-install.help

@@ -47,6 +47,7 @@ OFFICIAL PLUGINS
     - cloud-azure
     - cloud-gce
     - delete-by-query
+    - discovery-multicast
     - lang-javascript
     - lang-python
     - mapper-murmur3

core/src/test/java/org/elasticsearch/benchmark/transport/BenchmarkNettyLargeMessages.java

@@ -35,6 +35,7 @@ import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.*;
 import org.elasticsearch.transport.netty.NettyTransport;
 
+import java.net.InetAddress;
 import java.util.concurrent.CountDownLatch;
 
 import static org.elasticsearch.transport.TransportRequestOptions.options;
@@ -44,7 +45,7 @@ import static org.elasticsearch.transport.TransportRequestOptions.options;
  */
 public class BenchmarkNettyLargeMessages {
 
-    public static void main(String[] args) throws InterruptedException {
+    public static void main(String[] args) throws Exception {
         final ByteSizeValue payloadSize = new ByteSizeValue(10, ByteSizeUnit.MB);
         final int NUMBER_OF_ITERATIONS = 100000;
         final int NUMBER_OF_CLIENTS = 5;
@@ -63,7 +64,7 @@ public class BenchmarkNettyLargeMessages {
                 new NettyTransport(settings, threadPool, networkService, BigArrays.NON_RECYCLING_INSTANCE, Version.CURRENT, new NamedWriteableRegistry()), threadPool
         ).start();
 
-        final DiscoveryNode bigNode = new DiscoveryNode("big", new InetSocketTransportAddress("localhost", 9300), Version.CURRENT);
+        final DiscoveryNode bigNode = new DiscoveryNode("big", new InetSocketTransportAddress(InetAddress.getByName("localhost"), 9300), Version.CURRENT);
 //        final DiscoveryNode smallNode = new DiscoveryNode("small", new InetSocketTransportAddress("localhost", 9300));
         final DiscoveryNode smallNode = bigNode;
 

core/src/test/java/org/elasticsearch/benchmark/transport/netty/NettyEchoBenchmark.java

@@ -27,13 +27,14 @@ import org.jboss.netty.channel.*;
 import org.jboss.netty.channel.socket.nio.NioClientSocketChannelFactory;
 import org.jboss.netty.channel.socket.nio.NioServerSocketChannelFactory;
 
+import java.net.InetAddress;
 import java.net.InetSocketAddress;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.Executors;
 
 public class NettyEchoBenchmark {
 
-    public static void main(String[] args) {
+    public static void main(String[] args) throws Exception {
         final int payloadSize = 100;
         int CYCLE_SIZE = 50000;
         final long NUMBER_OF_ITERATIONS = 500000;
@@ -58,7 +59,7 @@ public class NettyEchoBenchmark {
         });
 
         // Bind and start to accept incoming connections.
-        serverBootstrap.bind(new InetSocketAddress(9000));
+        serverBootstrap.bind(new InetSocketAddress(InetAddress.getLoopbackAddress(), 9000));
 
         ClientBootstrap clientBootstrap = new ClientBootstrap(
                 new NioClientSocketChannelFactory(
@@ -78,7 +79,7 @@ public class NettyEchoBenchmark {
         });
 
         // Start the connection attempt.
-        ChannelFuture future = clientBootstrap.connect(new InetSocketAddress("localhost", 9000));
+        ChannelFuture future = clientBootstrap.connect(new InetSocketAddress(InetAddress.getLoopbackAddress(), 9000));
         future.awaitUninterruptibly();
         Channel clientChannel = future.getChannel();
 

core/src/test/java/org/elasticsearch/bwcompat/UnicastBackwardsCompatibilityIT.java

@@ -33,7 +33,6 @@ public class UnicastBackwardsCompatibilityIT extends ESBackcompatTestCase {
         return Settings.builder()
                 .put(super.nodeSettings(nodeOrdinal))
                 .put("transport.tcp.port", 9380 + nodeOrdinal)
-                .put("discovery.zen.ping.multicast.enabled", false)
                 .put("discovery.zen.ping.unicast.hosts", "localhost:9380,localhost:9381,localhost:9390,localhost:9391")
                 .build();
     }
@@ -43,7 +42,6 @@ public class UnicastBackwardsCompatibilityIT extends ESBackcompatTestCase {
         return Settings.settingsBuilder()
                 .put(super.externalNodeSettings(nodeOrdinal))
                 .put("transport.tcp.port", 9390 + nodeOrdinal)
-                .put("discovery.zen.ping.multicast.enabled", false)
                 .put("discovery.zen.ping.unicast.hosts", "localhost:9380,localhost:9381,localhost:9390,localhost:9391")
                 .build();
     }

core/src/test/java/org/elasticsearch/client/transport/FailAndRetryMockTransport.java

@@ -123,7 +123,7 @@ abstract class FailAndRetryMockTransport<Response extends TransportResponse> imp
     }
 
     @Override
-    public TransportAddress[] addressesFromString(String address) throws Exception {
+    public TransportAddress[] addressesFromString(String address, int perAddressLimit) throws Exception {
         throw new UnsupportedOperationException();
     }
 

core/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java

@@ -32,6 +32,8 @@ import org.elasticsearch.transport.*;
 import org.junit.Test;
 
 import java.io.Closeable;
+import java.util.Collections;
+import java.util.List;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -53,6 +55,11 @@ public class TransportClientNodesServiceTests extends ESTestCase {
         TestIteration() {
             threadPool = new ThreadPool("transport-client-nodes-service-tests");
             transport = new FailAndRetryMockTransport<TestResponse>(getRandom()) {
+                @Override
+                public List<String> getLocalAddresses() {
+                    return Collections.EMPTY_LIST;
+                }
+
                 @Override
                 protected TestResponse newResponse() {
                     return  new TestResponse();

core/src/test/java/org/elasticsearch/client/transport/TransportClientRetryIT.java

@@ -60,7 +60,7 @@ public class TransportClientRetryIT extends ESIntegTestCase {
 
         Settings.Builder builder = settingsBuilder().put("client.transport.nodes_sampler_interval", "1s")
                 .put("name", "transport_client_retry_test")
-                .put("node.mode", InternalTestCluster.nodeMode())
+                .put("node.mode", internalCluster().getNodeMode())
                 .put(ClusterName.SETTING, internalCluster().getClusterName())
                 .put(InternalSettingsPreparer.IGNORE_SYSTEM_PROPERTIES_SETTING, true)
                 .put("path.home", createTempDir());

core/src/test/java/org/elasticsearch/cluster/ClusterServiceIT.java

@@ -66,6 +66,7 @@ import static org.hamcrest.Matchers.notNullValue;
  *
  */
 @ClusterScope(scope = Scope.TEST, numDataNodes = 0)
+@ESIntegTestCase.SuppressLocalMode
 public class ClusterServiceIT extends ESIntegTestCase {
 
     @Test

core/src/test/java/org/elasticsearch/cluster/MinimumMasterNodesIT.java

@@ -44,6 +44,7 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitC
 import static org.hamcrest.Matchers.*;
 
 @ClusterScope(scope = Scope.TEST, numDataNodes = 0)
+@ESIntegTestCase.SuppressLocalMode
 public class MinimumMasterNodesIT extends ESIntegTestCase {
 
     @Test

core/src/test/java/org/elasticsearch/cluster/NoMasterNodeIT.java

@@ -57,6 +57,7 @@ import static org.hamcrest.Matchers.lessThan;
 /**
  */
 @ClusterScope(scope = Scope.TEST, numDataNodes = 0)
+@ESIntegTestCase.SuppressLocalMode
 public class NoMasterNodeIT extends ESIntegTestCase {
 
     @Test

core/src/test/java/org/elasticsearch/cluster/SpecificMasterNodesIT.java

@@ -34,6 +34,7 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcke
 import static org.hamcrest.Matchers.*;
 
 @ClusterScope(scope = Scope.TEST, numDataNodes = 0)
+@ESIntegTestCase.SuppressLocalMode
 public class SpecificMasterNodesIT extends ESIntegTestCase {
 
     protected final Settings.Builder settingsBuilder() {

core/src/test/java/org/elasticsearch/cluster/allocation/SimpleAllocationIT.java

@@ -18,12 +18,16 @@
  */
 package org.elasticsearch.cluster.allocation;
 
+import org.elasticsearch.cluster.ClusterInfoService;
+import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.InternalClusterInfoService;
 import org.elasticsearch.cluster.routing.RoutingNode;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.junit.Test;
 
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS;
+import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS;
 import static org.elasticsearch.common.settings.Settings.settingsBuilder;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.equalTo;

core/src/test/java/org/elasticsearch/cluster/routing/AllocationIdTests.java

@@ -35,7 +35,7 @@ public class AllocationIdTests extends ESTestCase {
         assertThat(shard.allocationId(), nullValue());
 
         logger.info("-- initialize the shard");
-        shard.initialize("node1");
+        shard.initialize("node1", -1);
         AllocationId allocationId = shard.allocationId();
         assertThat(allocationId, notNullValue());
         assertThat(allocationId.getId(), notNullValue());
@@ -53,12 +53,12 @@ public class AllocationIdTests extends ESTestCase {
     public void testSuccessfulRelocation() {
         logger.info("-- build started shard");
         ShardRouting shard = ShardRouting.newUnassigned("test", 0, null, true, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null));
-        shard.initialize("node1");
+        shard.initialize("node1", -1);
         shard.moveToStarted();
 
         AllocationId allocationId = shard.allocationId();
         logger.info("-- relocate the shard");
-        shard.relocate("node2");
+        shard.relocate("node2", -1);
         assertThat(shard.allocationId(), not(equalTo(allocationId)));
         assertThat(shard.allocationId().getId(), equalTo(allocationId.getId()));
         assertThat(shard.allocationId().getRelocationId(), notNullValue());
@@ -77,12 +77,12 @@ public class AllocationIdTests extends ESTestCase {
     public void testCancelRelocation() {
         logger.info("-- build started shard");
         ShardRouting shard = ShardRouting.newUnassigned("test", 0, null, true, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null));
-        shard.initialize("node1");
+        shard.initialize("node1", -1);
         shard.moveToStarted();
 
         AllocationId allocationId = shard.allocationId();
         logger.info("-- relocate the shard");
-        shard.relocate("node2");
+        shard.relocate("node2", -1);
         assertThat(shard.allocationId(), not(equalTo(allocationId)));
         assertThat(shard.allocationId().getId(), equalTo(allocationId.getId()));
         assertThat(shard.allocationId().getRelocationId(), notNullValue());
@@ -98,7 +98,7 @@ public class AllocationIdTests extends ESTestCase {
     public void testMoveToUnassigned() {
         logger.info("-- build started shard");
         ShardRouting shard = ShardRouting.newUnassigned("test", 0, null, true, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null));
-        shard.initialize("node1");
+        shard.initialize("node1", -1);
         shard.moveToStarted();
 
         logger.info("-- move to unassigned");
@@ -110,7 +110,7 @@ public class AllocationIdTests extends ESTestCase {
     public void testReinitializing() {
         logger.info("-- build started shard");
         ShardRouting shard = ShardRouting.newUnassigned("test", 0, null, true, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null));
-        shard.initialize("node1");
+        shard.initialize("node1", -1);
         shard.moveToStarted();
         AllocationId allocationId = shard.allocationId();
 

core/src/test/java/org/elasticsearch/cluster/routing/RandomShardRoutingMutator.java

@@ -42,7 +42,7 @@ public final class RandomShardRoutingMutator {
                 break;
             case 1:
                 if (shardRouting.unassigned()) {
-                    shardRouting.initialize(randomFrom(nodes));
+                    shardRouting.initialize(randomFrom(nodes), -1);
                 }
                 break;
             case 2:

core/src/test/java/org/elasticsearch/cluster/routing/ShardRoutingHelper.java

@@ -25,7 +25,11 @@ package org.elasticsearch.cluster.routing;
 public class ShardRoutingHelper {
 
     public static void relocate(ShardRouting routing, String nodeId) {
-        routing.relocate(nodeId);
+        relocate(routing, nodeId, -1);
+    }
+
+    public static void relocate(ShardRouting routing, String nodeId, long expectedByteSize) {
+        routing.relocate(nodeId, expectedByteSize);
     }
 
     public static void moveToStarted(ShardRouting routing) {
@@ -33,6 +37,10 @@ public class ShardRoutingHelper {
     }
 
     public static void initialize(ShardRouting routing, String nodeId) {
-        routing.initialize(nodeId);
+        initialize(routing, nodeId, -1);
+    }
+
+    public static void initialize(ShardRouting routing, String nodeId, long expectedSize) {
+        routing.initialize(nodeId, expectedSize);
     }
 }

core/src/test/java/org/elasticsearch/cluster/routing/ShardRoutingTests.java

@@ -103,12 +103,12 @@ public class ShardRoutingTests extends ESTestCase {
         ShardRouting startedShard1 = new ShardRouting(initializingShard1);
         startedShard1.moveToStarted();
         ShardRouting sourceShard0a = new ShardRouting(startedShard0);
-        sourceShard0a.relocate("node2");
+        sourceShard0a.relocate("node2", -1);
         ShardRouting targetShard0a = sourceShard0a.buildTargetRelocatingShard();
         ShardRouting sourceShard0b = new ShardRouting(startedShard0);
-        sourceShard0b.relocate("node2");
+        sourceShard0b.relocate("node2", -1);
         ShardRouting sourceShard1 = new ShardRouting(startedShard1);
-        sourceShard1.relocate("node2");
+        sourceShard1.relocate("node2", -1);
 
         // test true scenarios
         assertTrue(targetShard0a.isRelocationTargetOf(sourceShard0a));
@@ -254,7 +254,7 @@ public class ShardRoutingTests extends ESTestCase {
             }
 
             try {
-                routing.initialize("boom");
+                routing.initialize("boom", -1);
                 fail("must be frozen");
             } catch (IllegalStateException ex) {
                 // expected
@@ -273,7 +273,7 @@ public class ShardRoutingTests extends ESTestCase {
             }
 
             try {
-                routing.relocate("foobar");
+                routing.relocate("foobar", -1);
                 fail("must be frozen");
             } catch (IllegalStateException ex) {
                 // expected
@@ -287,4 +287,39 @@ public class ShardRoutingTests extends ESTestCase {
             assertEquals(version, routing.version());
         }
     }
+
+    public void testExpectedSize() throws IOException {
+        final int iters = randomIntBetween(10, 100);
+        for (int i = 0; i < iters; i++) {
+            ShardRouting routing = randomShardRouting("test", 0);
+            long byteSize = randomIntBetween(0, Integer.MAX_VALUE);
+            if (routing.unassigned()) {
+                ShardRoutingHelper.initialize(routing, "foo", byteSize);
+            } else if (routing.started()) {
+                ShardRoutingHelper.relocate(routing, "foo", byteSize);
+            } else {
+                byteSize = -1;
+            }
+            if (randomBoolean()) {
+                BytesStreamOutput out = new BytesStreamOutput();
+                routing.writeTo(out);
+                routing = ShardRouting.readShardRoutingEntry(StreamInput.wrap(out.bytes()));
+            }
+            if (routing.initializing() || routing.relocating()) {
+                assertEquals(routing.toString(), byteSize, routing.getExpectedShardSize());
+                if (byteSize >= 0) {
+                    assertTrue(routing.toString(), routing.toString().contains("expected_shard_size[" + byteSize + "]"));
+                }
+                if (routing.initializing()) {
+                    routing = new ShardRouting(routing);
+                    routing.moveToStarted();
+                    assertEquals(-1, routing.getExpectedShardSize());
+                    assertFalse(routing.toString(), routing.toString().contains("expected_shard_size[" + byteSize + "]"));
+                }
+            } else {
+                assertFalse(routing.toString(), routing.toString().contains("expected_shard_size [" + byteSize + "]"));
+                assertEquals(byteSize, routing.getExpectedShardSize());
+            }
+        }
+    }
 }

core/src/test/java/org/elasticsearch/cluster/routing/TestShardRouting.java

@@ -28,25 +28,25 @@ import org.elasticsearch.test.ESTestCase;
 public class TestShardRouting {
 
     public static ShardRouting newShardRouting(String index, int shardId, String currentNodeId, boolean primary, ShardRoutingState state, long version) {
-        return new ShardRouting(index, shardId, currentNodeId, null, null, primary, state, version, buildUnassignedInfo(state), buildAllocationId(state), true);
+        return new ShardRouting(index, shardId, currentNodeId, null, null, primary, state, version, buildUnassignedInfo(state), buildAllocationId(state), true, -1);
     }
 
     public static ShardRouting newShardRouting(String index, int shardId, String currentNodeId, String relocatingNodeId, boolean primary, ShardRoutingState state, long version) {
-        return new ShardRouting(index, shardId, currentNodeId, relocatingNodeId, null, primary, state, version, buildUnassignedInfo(state), buildAllocationId(state), true);
+        return new ShardRouting(index, shardId, currentNodeId, relocatingNodeId, null, primary, state, version, buildUnassignedInfo(state), buildAllocationId(state), true, -1);
     }
 
     public static ShardRouting newShardRouting(String index, int shardId, String currentNodeId, String relocatingNodeId, boolean primary, ShardRoutingState state, AllocationId allocationId, long version) {
-        return new ShardRouting(index, shardId, currentNodeId, relocatingNodeId, null, primary, state, version, buildUnassignedInfo(state), allocationId, true);
+        return new ShardRouting(index, shardId, currentNodeId, relocatingNodeId, null, primary, state, version, buildUnassignedInfo(state), allocationId, true, -1);
     }
 
     public static ShardRouting newShardRouting(String index, int shardId, String currentNodeId, String relocatingNodeId, RestoreSource restoreSource, boolean primary, ShardRoutingState state, long version) {
-        return new ShardRouting(index, shardId, currentNodeId, relocatingNodeId, restoreSource, primary, state, version, buildUnassignedInfo(state), buildAllocationId(state), true);
+        return new ShardRouting(index, shardId, currentNodeId, relocatingNodeId, restoreSource, primary, state, version, buildUnassignedInfo(state), buildAllocationId(state), true, -1);
     }
 
     public static ShardRouting newShardRouting(String index, int shardId, String currentNodeId,
                                                String relocatingNodeId, RestoreSource restoreSource, boolean primary, ShardRoutingState state, long version,
                                                UnassignedInfo unassignedInfo) {
-        return new ShardRouting(index, shardId, currentNodeId, relocatingNodeId, restoreSource, primary, state, version, unassignedInfo, buildAllocationId(state), true);
+        return new ShardRouting(index, shardId, currentNodeId, relocatingNodeId, restoreSource, primary, state, version, unassignedInfo, buildAllocationId(state), true, -1);
     }
 
     private static AllocationId buildAllocationId(ShardRoutingState state) {

core/src/test/java/org/elasticsearch/cluster/routing/UnassignedInfoTests.java

@@ -192,7 +192,7 @@ public class UnassignedInfoTests extends ESAllocationTestCase {
         ShardRouting shard = TestShardRouting.newShardRouting("test", 1, null, null, null, true, ShardRoutingState.UNASSIGNED, 1, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null));
         ShardRouting mutable = new ShardRouting(shard);
         assertThat(mutable.unassignedInfo(), notNullValue());
-        mutable.initialize("test_node");
+        mutable.initialize("test_node", -1);
         assertThat(mutable.state(), equalTo(ShardRoutingState.INITIALIZING));
         assertThat(mutable.unassignedInfo(), notNullValue());
         mutable.moveToStarted();

core/src/test/java/org/elasticsearch/cluster/routing/allocation/BalanceConfigurationTests.java

@@ -369,37 +369,37 @@ public class BalanceConfigurationTests extends ESAllocationTestCase {
                     switch (sr.id()) {
                         case 0:
                             if (sr.primary()) {
-                                allocation.routingNodes().initialize(sr, "node1");
+                                allocation.routingNodes().initialize(sr, "node1", -1);
                             } else {
-                                allocation.routingNodes().initialize(sr, "node0");
+                                allocation.routingNodes().initialize(sr, "node0", -1);
                             }
                             break;
                         case 1:
                             if (sr.primary()) {
-                                allocation.routingNodes().initialize(sr, "node1");
+                                allocation.routingNodes().initialize(sr, "node1", -1);
                             } else {
-                                allocation.routingNodes().initialize(sr, "node2");
+                                allocation.routingNodes().initialize(sr, "node2", -1);
                             }
                             break;
                         case 2:
                             if (sr.primary()) {
-                                allocation.routingNodes().initialize(sr, "node3");
+                                allocation.routingNodes().initialize(sr, "node3", -1);
                             } else {
-                                allocation.routingNodes().initialize(sr, "node2");
+                                allocation.routingNodes().initialize(sr, "node2", -1);
                             }
                             break;
                         case 3:
                             if (sr.primary()) {
-                                allocation.routingNodes().initialize(sr, "node3");
+                                allocation.routingNodes().initialize(sr, "node3", -1);
                             } else {
-                                allocation.routingNodes().initialize(sr, "node1");
+                                allocation.routingNodes().initialize(sr, "node1", -1);
                             }
                             break;
                         case 4:
                             if (sr.primary()) {
-                                allocation.routingNodes().initialize(sr, "node2");
+                                allocation.routingNodes().initialize(sr, "node2", -1);
                             } else {
-                                allocation.routingNodes().initialize(sr, "node0");
+                                allocation.routingNodes().initialize(sr, "node0", -1);
                             }
                             break;
                     }

core/src/test/java/org/elasticsearch/cluster/routing/allocation/ExpectedShardSizeAllocationTests.java

@@ -0,0 +1,179 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cluster.routing.allocation;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.ClusterInfo;
+import org.elasticsearch.cluster.ClusterInfoService;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.routing.RoutingNodes;
+import org.elasticsearch.cluster.routing.RoutingTable;
+import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.routing.ShardRoutingState;
+import org.elasticsearch.cluster.routing.allocation.command.AllocationCommands;
+import org.elasticsearch.cluster.routing.allocation.command.MoveAllocationCommand;
+import org.elasticsearch.cluster.routing.allocation.decider.ShardsLimitAllocationDecider;
+import org.elasticsearch.common.logging.ESLogger;
+import org.elasticsearch.common.logging.Loggers;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.test.ESAllocationTestCase;
+import org.junit.Test;
+
+import java.util.Collections;
+
+import static org.elasticsearch.cluster.routing.ShardRoutingState.*;
+import static org.elasticsearch.cluster.routing.allocation.RoutingNodesUtils.numberOfShardsOfType;
+import static org.elasticsearch.common.settings.Settings.settingsBuilder;
+import static org.hamcrest.Matchers.equalTo;
+
+/**
+ */
+public class ExpectedShardSizeAllocationTests extends ESAllocationTestCase {
+
+    private final ESLogger logger = Loggers.getLogger(ExpectedShardSizeAllocationTests.class);
+
+    @Test
+    public void testInitializingHasExpectedSize() {
+        final long byteSize = randomIntBetween(0, Integer.MAX_VALUE);
+        AllocationService strategy = createAllocationService(Settings.EMPTY, new ClusterInfoService() {
+            @Override
+            public ClusterInfo getClusterInfo() {
+                return new ClusterInfo(Collections.EMPTY_MAP, Collections.EMPTY_MAP) {
+                    @Override
+                    public Long getShardSize(ShardRouting shardRouting) {
+                        if (shardRouting.index().equals("test") && shardRouting.shardId().getId() == 0) {
+                            return byteSize;
+                        }
+                        return null;
+                    }
+                };
+            }
+
+            @Override
+            public void addListener(Listener listener) {
+            }
+        });
+
+        logger.info("Building initial routing table");
+
+        MetaData metaData = MetaData.builder()
+                .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)
+                        .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+                        .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1)))
+                .build();
+
+        RoutingTable routingTable = RoutingTable.builder()
+                .addAsNew(metaData.index("test"))
+                .build();
+
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+        logger.info("Adding one node and performing rerouting");
+        clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1"))).build();
+        routingTable = strategy.reroute(clusterState).routingTable();
+        clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
+
+        assertEquals(1, clusterState.getRoutingNodes().node("node1").numberOfShardsWithState(ShardRoutingState.INITIALIZING));
+        assertEquals(byteSize, clusterState.getRoutingNodes().getRoutingTable().shardsWithState(ShardRoutingState.INITIALIZING).get(0).getExpectedShardSize());
+        logger.info("Start the primary shard");
+        RoutingNodes routingNodes = clusterState.getRoutingNodes();
+        routingTable = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING)).routingTable();
+        clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
+
+        assertEquals(1, clusterState.getRoutingNodes().node("node1").numberOfShardsWithState(ShardRoutingState.STARTED));
+        assertEquals(1, clusterState.getRoutingNodes().unassigned().size());
+
+        logger.info("Add another one node and reroute");
+        clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).put(newNode("node2"))).build();
+        routingTable = strategy.reroute(clusterState).routingTable();
+        clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
+
+        assertEquals(1, clusterState.getRoutingNodes().node("node2").numberOfShardsWithState(ShardRoutingState.INITIALIZING));
+        assertEquals(byteSize, clusterState.getRoutingNodes().getRoutingTable().shardsWithState(ShardRoutingState.INITIALIZING).get(0).getExpectedShardSize());
+    }
+
+    @Test
+    public void testExpectedSizeOnMove() {
+        final long byteSize = randomIntBetween(0, Integer.MAX_VALUE);
+        final AllocationService allocation = createAllocationService(Settings.EMPTY, new ClusterInfoService() {
+            @Override
+            public ClusterInfo getClusterInfo() {
+                return new ClusterInfo(Collections.EMPTY_MAP, Collections.EMPTY_MAP) {
+                    @Override
+                    public Long getShardSize(ShardRouting shardRouting) {
+                        if (shardRouting.index().equals("test") && shardRouting.shardId().getId() == 0) {
+                            return byteSize;
+                        }
+                        return null;
+                    }
+                };
+            }
+
+            @Override
+            public void addListener(Listener listener) {
+            }
+        });
+        logger.info("creating an index with 1 shard, no replica");
+        MetaData metaData = MetaData.builder()
+                .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(0))
+                .build();
+        RoutingTable routingTable = RoutingTable.builder()
+                .addAsNew(metaData.index("test"))
+                .build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+
+        logger.info("adding two nodes and performing rerouting");
+        clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2"))).build();
+        RoutingAllocation.Result rerouteResult = allocation.reroute(clusterState);
+        clusterState = ClusterState.builder(clusterState).routingTable(rerouteResult.routingTable()).build();
+
+        logger.info("start primary shard");
+        rerouteResult = allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
+        clusterState = ClusterState.builder(clusterState).routingTable(rerouteResult.routingTable()).build();
+
+        logger.info("move the shard");
+        String existingNodeId = clusterState.routingTable().index("test").shard(0).primaryShard().currentNodeId();
+        String toNodeId;
+        if ("node1".equals(existingNodeId)) {
+            toNodeId = "node2";
+        } else {
+            toNodeId = "node1";
+        }
+        rerouteResult = allocation.reroute(clusterState, new AllocationCommands(new MoveAllocationCommand(new ShardId("test", 0), existingNodeId, toNodeId)));
+        assertThat(rerouteResult.changed(), equalTo(true));
+        clusterState = ClusterState.builder(clusterState).routingTable(rerouteResult.routingTable()).build();
+        assertEquals(clusterState.getRoutingNodes().node(existingNodeId).get(0).state(), ShardRoutingState.RELOCATING);
+        assertEquals(clusterState.getRoutingNodes().node(toNodeId).get(0).state(),ShardRoutingState.INITIALIZING);
+
+        assertEquals(clusterState.getRoutingNodes().node(existingNodeId).get(0).getExpectedShardSize(), byteSize);
+        assertEquals(clusterState.getRoutingNodes().node(toNodeId).get(0).getExpectedShardSize(), byteSize);
+
+        logger.info("finish moving the shard");
+        rerouteResult = allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
+        clusterState = ClusterState.builder(clusterState).routingTable(rerouteResult.routingTable()).build();
+
+        assertThat(clusterState.getRoutingNodes().node(existingNodeId).isEmpty(), equalTo(true));
+        assertThat(clusterState.getRoutingNodes().node(toNodeId).get(0).state(), equalTo(ShardRoutingState.STARTED));
+        assertEquals(clusterState.getRoutingNodes().node(toNodeId).get(0).getExpectedShardSize(), -1);
+    }
+}

core/src/test/java/org/elasticsearch/cluster/routing/allocation/RebalanceAfterActiveTests.java

@@ -20,6 +20,8 @@
 package org.elasticsearch.cluster.routing.allocation;
 
 import org.elasticsearch.Version;
+import org.elasticsearch.cluster.ClusterInfo;
+import org.elasticsearch.cluster.ClusterInfoService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
@@ -27,12 +29,16 @@ import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.RoutingNode;
 import org.elasticsearch.cluster.routing.RoutingNodes;
 import org.elasticsearch.cluster.routing.RoutingTable;
+import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.allocation.decider.ClusterRebalanceAllocationDecider;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.logging.Loggers;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.ESAllocationTestCase;
 import org.junit.Test;
 
+import java.util.Collections;
+
 import static org.elasticsearch.cluster.routing.ShardRoutingState.*;
 import static org.elasticsearch.common.settings.Settings.settingsBuilder;
 import static org.hamcrest.Matchers.equalTo;
@@ -47,12 +53,33 @@ public class RebalanceAfterActiveTests extends ESAllocationTestCase {
 
     @Test
     public void testRebalanceOnlyAfterAllShardsAreActive() {
-        AllocationService strategy = createAllocationService(settingsBuilder()
-                .put("cluster.routing.allocation.concurrent_recoveries", 10)
-                .put(ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE, "always")
-                .put("cluster.routing.allocation.cluster_concurrent_rebalance", -1)
-                .build());
+        final long[] sizes = new long[5];
+        for (int i =0; i < sizes.length; i++) {
+            sizes[i] = randomIntBetween(0, Integer.MAX_VALUE);
+        }
 
+        AllocationService strategy = createAllocationService(settingsBuilder()
+                        .put("cluster.routing.allocation.concurrent_recoveries", 10)
+                        .put(ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE, "always")
+                        .put("cluster.routing.allocation.cluster_concurrent_rebalance", -1)
+                        .build(),
+                new ClusterInfoService() {
+                    @Override
+                    public ClusterInfo getClusterInfo() {
+                        return new ClusterInfo(Collections.EMPTY_MAP, Collections.EMPTY_MAP) {
+                            @Override
+                            public Long getShardSize(ShardRouting shardRouting) {
+                                if (shardRouting.index().equals("test")) {
+                                    return sizes[shardRouting.getId()];
+                                }
+                                return null;                    }
+                        };
+                    }
+
+                    @Override
+                    public void addListener(Listener listener) {
+                    }
+                });
         logger.info("Building initial routing table");
 
         MetaData metaData = MetaData.builder()
@@ -97,6 +124,7 @@ public class RebalanceAfterActiveTests extends ESAllocationTestCase {
             assertThat(routingTable.index("test").shard(i).shards().size(), equalTo(2));
             assertThat(routingTable.index("test").shard(i).primaryShard().state(), equalTo(STARTED));
             assertThat(routingTable.index("test").shard(i).replicaShards().get(0).state(), equalTo(INITIALIZING));
+            assertEquals(routingTable.index("test").shard(i).replicaShards().get(0).getExpectedShardSize(), sizes[i]);
         }
 
         logger.info("now, start 8 more nodes, and check that no rebalancing/relocation have happened");
@@ -112,6 +140,8 @@ public class RebalanceAfterActiveTests extends ESAllocationTestCase {
             assertThat(routingTable.index("test").shard(i).shards().size(), equalTo(2));
             assertThat(routingTable.index("test").shard(i).primaryShard().state(), equalTo(STARTED));
             assertThat(routingTable.index("test").shard(i).replicaShards().get(0).state(), equalTo(INITIALIZING));
+            assertEquals(routingTable.index("test").shard(i).replicaShards().get(0).getExpectedShardSize(), sizes[i]);
+
         }
 
         logger.info("start the replica shards, rebalancing should start");
@@ -124,6 +154,16 @@ public class RebalanceAfterActiveTests extends ESAllocationTestCase {
         // we only allow one relocation at a time
         assertThat(routingTable.shardsWithState(STARTED).size(), equalTo(5));
         assertThat(routingTable.shardsWithState(RELOCATING).size(), equalTo(5));
+        for (int i = 0; i < routingTable.index("test").shards().size(); i++) {
+            int num = 0;
+            for (ShardRouting routing : routingTable.index("test").shard(i).shards()) {
+                if (routing.state() == RELOCATING || routing.state() == INITIALIZING) {
+                    assertEquals(routing.getExpectedShardSize(), sizes[i]);
+                    num++;
+                }
+            }
+            assertTrue(num > 0);
+        }
 
         logger.info("complete relocation, other half of relocation should happen");
         routingNodes = clusterState.getRoutingNodes();
@@ -135,6 +175,14 @@ public class RebalanceAfterActiveTests extends ESAllocationTestCase {
         // we now only relocate 3, since 2 remain where they are!
         assertThat(routingTable.shardsWithState(STARTED).size(), equalTo(7));
         assertThat(routingTable.shardsWithState(RELOCATING).size(), equalTo(3));
+        for (int i = 0; i < routingTable.index("test").shards().size(); i++) {
+            for (ShardRouting routing : routingTable.index("test").shard(i).shards()) {
+                if (routing.state() == RELOCATING || routing.state() == INITIALIZING) {
+                    assertEquals(routing.getExpectedShardSize(), sizes[i]);
+                }
+            }
+        }
+
 
         logger.info("complete relocation, thats it!");
         routingNodes = clusterState.getRoutingNodes();

core/src/test/java/org/elasticsearch/common/network/NetworkAddressTests.java

@@ -0,0 +1,99 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.network;
+
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.net.Inet6Address;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+
+/**
+ * Tests for network address formatting. Please avoid using any methods that cause DNS lookups!
+ */
+public class NetworkAddressTests extends ESTestCase {
+    
+    public void testFormatV4() throws Exception {
+        assertEquals("localhost/127.0.0.1", NetworkAddress.format(forge("localhost", "127.0.0.1")));
+        assertEquals("127.0.0.1", NetworkAddress.format(forge(null, "127.0.0.1")));
+    }
+    
+    public void testFormatV6() throws Exception {
+        assertEquals("localhost/::1", NetworkAddress.format(forge("localhost", "::1")));
+        assertEquals("::1", NetworkAddress.format(forge(null, "::1")));
+    }
+    
+    public void testFormatAddressV4() throws Exception {
+        assertEquals("127.0.0.1", NetworkAddress.formatAddress(forge("localhost", "127.0.0.1")));
+        assertEquals("127.0.0.1", NetworkAddress.formatAddress(forge(null, "127.0.0.1")));
+    }
+    
+    public void testFormatAddressV6() throws Exception {
+        assertEquals("::1", NetworkAddress.formatAddress(forge("localhost", "::1")));
+        assertEquals("::1", NetworkAddress.formatAddress(forge(null, "::1")));
+    }
+    
+    public void testFormatPortV4() throws Exception {
+        assertEquals("localhost/127.0.0.1:1234", NetworkAddress.format(new InetSocketAddress(forge("localhost", "127.0.0.1"), 1234)));
+        assertEquals("127.0.0.1:1234", NetworkAddress.format(new InetSocketAddress(forge(null, "127.0.0.1"), 1234)));
+    }
+    
+    public void testFormatPortV6() throws Exception {
+        assertEquals("localhost/[::1]:1234", NetworkAddress.format(new InetSocketAddress(forge("localhost", "::1"), 1234)));
+        assertEquals("[::1]:1234",NetworkAddress.format(new InetSocketAddress(forge(null, "::1"), 1234)));
+    }
+    
+    public void testFormatAddressPortV4() throws Exception {
+        assertEquals("127.0.0.1:1234", NetworkAddress.formatAddress(new InetSocketAddress(forge("localhost", "127.0.0.1"), 1234)));
+        assertEquals("127.0.0.1:1234", NetworkAddress.formatAddress(new InetSocketAddress(forge(null, "127.0.0.1"), 1234)));
+    }
+    
+    public void testFormatAddressPortV6() throws Exception {
+        assertEquals("[::1]:1234", NetworkAddress.formatAddress(new InetSocketAddress(forge("localhost", "::1"), 1234)));
+        assertEquals("[::1]:1234", NetworkAddress.formatAddress(new InetSocketAddress(forge(null, "::1"), 1234)));
+    }
+    
+    public void testNoScopeID() throws Exception {
+        assertEquals("::1", NetworkAddress.format(forgeScoped(null, "::1", 5)));
+        assertEquals("localhost/::1", NetworkAddress.format(forgeScoped("localhost", "::1", 5)));
+        
+        assertEquals("::1", NetworkAddress.formatAddress(forgeScoped(null, "::1", 5)));
+        assertEquals("::1", NetworkAddress.formatAddress(forgeScoped("localhost", "::1", 5)));
+        
+        assertEquals("[::1]:1234", NetworkAddress.format(new InetSocketAddress(forgeScoped(null, "::1", 5), 1234)));
+        assertEquals("localhost/[::1]:1234", NetworkAddress.format(new InetSocketAddress(forgeScoped("localhost", "::1", 5), 1234)));
+        
+        assertEquals("[::1]:1234", NetworkAddress.formatAddress(new InetSocketAddress(forgeScoped(null, "::1", 5), 1234)));
+        assertEquals("[::1]:1234", NetworkAddress.formatAddress(new InetSocketAddress(forgeScoped("localhost", "::1", 5), 1234)));
+    }
+    
+    /** creates address without any lookups. hostname can be null, for missing */
+    private InetAddress forge(String hostname, String address) throws IOException {
+        byte bytes[] = InetAddress.getByName(address).getAddress();
+        return InetAddress.getByAddress(hostname, bytes);
+    }
+    
+    /** creates scoped ipv6 address without any lookups. hostname can be null, for missing */
+    private InetAddress forgeScoped(String hostname, String address, int scopeid) throws IOException {
+        byte bytes[] = InetAddress.getByName(address).getAddress();
+        return Inet6Address.getByAddress(hostname, bytes, scopeid);
+    }
+}

core/src/test/java/org/elasticsearch/discovery/DiscoveryWithServiceDisruptionsIT.java

@@ -72,6 +72,7 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcke
 import static org.hamcrest.Matchers.*;
 
 @ClusterScope(scope = Scope.TEST, numDataNodes = 0, transportClientRatio = 0)
+@ESIntegTestCase.SuppressLocalMode
 public class DiscoveryWithServiceDisruptionsIT extends ESIntegTestCase {
 
     private static final TimeValue DISRUPTION_HEALING_OVERHEAD = TimeValue.timeValueSeconds(40); // we use 30s as timeout in many places.
@@ -104,23 +105,10 @@ public class DiscoveryWithServiceDisruptionsIT extends ESIntegTestCase {
     }
 
     private List<String> startCluster(int numberOfNodes, int minimumMasterNode) throws ExecutionException, InterruptedException {
-        configureCluster(numberOfNodes, minimumMasterNode);
-        List<String> nodes = internalCluster().startNodesAsync(numberOfNodes).get();
-        ensureStableCluster(numberOfNodes);
-
-        // TODO: this is a temporary solution so that nodes will not base their reaction to a partition based on previous successful results
-        for (ZenPingService pingService : internalCluster().getInstances(ZenPingService.class)) {
-            for (ZenPing zenPing : pingService.zenPings()) {
-                if (zenPing instanceof UnicastZenPing) {
-                    ((UnicastZenPing) zenPing).clearTemporalResponses();
-                }
-            }
-        }
-        return nodes;
+        return startCluster(numberOfNodes, minimumMasterNode, null);
     }
 
-
-    private List<String> startUnicastCluster(int numberOfNodes, @Nullable int[] unicastHostsOrdinals, int minimumMasterNode) throws ExecutionException, InterruptedException {
+    private List<String> startCluster(int numberOfNodes, int minimumMasterNode, @Nullable int[] unicastHostsOrdinals) throws ExecutionException, InterruptedException {
         configureUnicastCluster(numberOfNodes, unicastHostsOrdinals, minimumMasterNode);
         List<String> nodes = internalCluster().startNodesAsync(numberOfNodes).get();
         ensureStableCluster(numberOfNodes);
@@ -142,38 +130,18 @@ public class DiscoveryWithServiceDisruptionsIT extends ESIntegTestCase {
             .put("discovery.zen.join_timeout", "10s")  // still long to induce failures but to long so test won't time out
             .put(DiscoverySettings.PUBLISH_TIMEOUT, "1s") // <-- for hitting simulated network failures quickly
             .put("http.enabled", false) // just to make test quicker
+            .put("transport.host", "127.0.0.1") // only bind on one IF we use v4 here by default
+            .put("transport.bind_host", "127.0.0.1")
+            .put("transport.publish_host", "127.0.0.1")
             .put("gateway.local.list_timeout", "10s") // still long to induce failures but to long so test won't time out
             .put("plugin.types", MockTransportService.TestPlugin.class.getName())
             .build();
 
-    private void configureCluster(int numberOfNodes, int minimumMasterNode) throws ExecutionException, InterruptedException {
-        if (randomBoolean()) {
-            configureMulticastCluster(numberOfNodes, minimumMasterNode);
-        } else {
-            configureUnicastCluster(numberOfNodes, null, minimumMasterNode);
-        }
-
-    }
-
-    private void configureMulticastCluster(int numberOfNodes, int minimumMasterNode) throws ExecutionException, InterruptedException {
-        if (minimumMasterNode < 0) {
-            minimumMasterNode = numberOfNodes / 2 + 1;
-        }
-        // TODO: Rarely use default settings form some of these
-        Settings settings = Settings.builder()
-                .put(DEFAULT_SETTINGS)
-                .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES, minimumMasterNode)
-                .build();
-
-        if (discoveryConfig == null) {
-            discoveryConfig = new ClusterDiscoveryConfiguration(numberOfNodes, settings);
-        }
-    }
-
     private void configureUnicastCluster(int numberOfNodes, @Nullable int[] unicastHostsOrdinals, int minimumMasterNode) throws ExecutionException, InterruptedException {
         if (minimumMasterNode < 0) {
             minimumMasterNode = numberOfNodes / 2 + 1;
         }
+        logger.info("---> configured unicast");
         // TODO: Rarely use default settings form some of these
         Settings nodeSettings = Settings.builder()
                 .put(DEFAULT_SETTINGS)
@@ -556,9 +524,7 @@ public class DiscoveryWithServiceDisruptionsIT extends ESIntegTestCase {
      */
     @Test
     public void testMasterNodeGCs() throws Exception {
-        // TODO: on mac OS multicast threads are shared between nodes and we therefore we can't simulate GC and stop pinging for just one node
-        // find a way to block thread creation in the generic thread pool to avoid this.
-        List<String> nodes = startUnicastCluster(3, null, -1);
+        List<String> nodes = startCluster(3, -1);
 
         String oldMasterNode = internalCluster().getMasterName();
         // a very long GC, but it's OK as we remove the disruption when it has had an effect
@@ -600,10 +566,8 @@ public class DiscoveryWithServiceDisruptionsIT extends ESIntegTestCase {
      */
     @Test
     public void testStaleMasterNotHijackingMajority() throws Exception {
-        // TODO: on mac OS multicast threads are shared between nodes and we therefore we can't simulate GC and stop pinging for just one node
-        // find a way to block thread creation in the generic thread pool to avoid this.
         // 3 node cluster with unicast discovery and minimum_master_nodes set to 2:
-        final List<String> nodes = startUnicastCluster(3, null, 2);
+        final List<String> nodes = startCluster(3, 2);
 
         // Save the current master node as old master node, because that node will get frozen
         final String oldMasterNode = internalCluster().getMasterName();
@@ -770,7 +734,7 @@ public class DiscoveryWithServiceDisruptionsIT extends ESIntegTestCase {
      */
     @Test
     public void unicastSinglePingResponseContainsMaster() throws Exception {
-        List<String> nodes = startUnicastCluster(4, new int[]{0}, -1);
+        List<String> nodes = startCluster(4, -1, new int[] {0});
         // Figure out what is the elected master node
         final String masterNode = internalCluster().getMasterName();
         logger.info("---> legit elected master node=" + masterNode);
@@ -807,7 +771,7 @@ public class DiscoveryWithServiceDisruptionsIT extends ESIntegTestCase {
     @Test
     @TestLogging("discovery.zen:TRACE,cluster.service:TRACE")
     public void isolatedUnicastNodes() throws Exception {
-        List<String> nodes = startUnicastCluster(4, new int[]{0}, -1);
+        List<String> nodes = startCluster(4, -1, new int[]{0});
         // Figure out what is the elected master node
         final String unicastTarget = nodes.get(0);
 
@@ -890,7 +854,7 @@ public class DiscoveryWithServiceDisruptionsIT extends ESIntegTestCase {
 
     @Test
     public void testClusterFormingWithASlowNode() throws Exception {
-        configureCluster(3, 2);
+        configureUnicastCluster(3, null, 2);
 
         SlowClusterStateProcessing disruption = new SlowClusterStateProcessing(getRandom(), 0, 0, 1000, 2000);
 
@@ -951,7 +915,7 @@ public class DiscoveryWithServiceDisruptionsIT extends ESIntegTestCase {
     @Test
     public void testIndexImportedFromDataOnlyNodesIfMasterLostDataFolder() throws Exception {
         // test for https://github.com/elastic/elasticsearch/issues/8823
-        configureCluster(2, 1);
+        configureUnicastCluster(2, null, 1);
         String masterNode = internalCluster().startMasterOnlyNode(Settings.EMPTY);
         internalCluster().startDataOnlyNode(Settings.EMPTY);
 
@@ -974,7 +938,7 @@ public class DiscoveryWithServiceDisruptionsIT extends ESIntegTestCase {
     @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/11665")
     @Test
     public void testIndicesDeleted() throws Exception {
-        configureCluster(3, 2);
+        configureUnicastCluster(3, null, 2);
         Future<List<String>> masterNodes= internalCluster().startMasterOnlyNodesAsync(2);
         Future<String> dataNode = internalCluster().startDataOnlyNodeAsync();
         dataNode.get();

core/src/test/java/org/elasticsearch/discovery/ZenUnicastDiscoveryIT.java

@@ -35,6 +35,7 @@ import java.util.concurrent.ExecutionException;
 import static org.hamcrest.Matchers.equalTo;
 
 @ClusterScope(scope = Scope.TEST, numDataNodes = 0)
+@ESIntegTestCase.SuppressLocalMode
 public class ZenUnicastDiscoveryIT extends ESIntegTestCase {
 
     private ClusterDiscoveryConfiguration discoveryConfig;

core/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryIT.java

@@ -32,6 +32,7 @@ import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.InetSocketTransportAddress;
 import org.elasticsearch.common.transport.LocalTransportAddress;
 import org.elasticsearch.discovery.Discovery;
 import org.elasticsearch.discovery.zen.elect.ElectMasterService;
@@ -41,15 +42,13 @@ import org.elasticsearch.discovery.zen.publish.PublishClusterStateAction;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.transport.BytesTransportRequest;
-import org.elasticsearch.transport.EmptyTransportResponseHandler;
-import org.elasticsearch.transport.TransportException;
-import org.elasticsearch.transport.TransportResponse;
-import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.transport.*;
 import org.hamcrest.Matchers;
 import org.junit.Test;
 
 import java.io.IOException;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
@@ -66,6 +65,7 @@ import static org.hamcrest.Matchers.nullValue;
 import static org.hamcrest.Matchers.sameInstance;
 
 @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, numClientNodes = 0)
+@ESIntegTestCase.SuppressLocalMode
 public class ZenDiscoveryIT extends ESIntegTestCase {
 
     @Test
@@ -226,15 +226,14 @@ public class ZenDiscoveryIT extends ESIntegTestCase {
     }
 
     @Test
-    public void testHandleNodeJoin_incompatibleMinVersion() {
+    public void testHandleNodeJoin_incompatibleMinVersion() throws UnknownHostException {
         Settings nodeSettings = Settings.settingsBuilder()
                 .put("discovery.type", "zen") // <-- To override the local setting if set externally
-                .put("node.mode", "local")  // <-- force local transport so we can fake a network address
                 .build();
         String nodeName = internalCluster().startNode(nodeSettings, Version.V_2_0_0_beta1);
         ZenDiscovery zenDiscovery = (ZenDiscovery) internalCluster().getInstance(Discovery.class, nodeName);
 
-        DiscoveryNode node = new DiscoveryNode("_node_id", new LocalTransportAddress("_id"), Version.V_1_6_0);
+        DiscoveryNode node = new DiscoveryNode("_node_id", new InetSocketTransportAddress(InetAddress.getByName("0.0.0.0"), 0), Version.V_1_6_0);
         final AtomicReference<IllegalStateException> holder = new AtomicReference<>();
         zenDiscovery.handleJoinRequest(node, new MembershipAction.JoinCallback() {
             @Override

core/src/test/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPingIT.java

@@ -24,6 +24,7 @@ import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.network.NetworkAddress;
 import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.InetSocketTransportAddress;
@@ -39,6 +40,8 @@ import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.transport.netty.NettyTransport;
 import org.junit.Test;
 
+import java.net.InetSocketAddress;
+
 import static org.hamcrest.Matchers.equalTo;
 
 public class UnicastZenPingIT extends ESTestCase {
@@ -68,8 +71,8 @@ public class UnicastZenPingIT extends ESTestCase {
         InetSocketTransportAddress addressB = (InetSocketTransportAddress) transportB.boundAddress().publishAddress();
 
         Settings hostsSettings = Settings.settingsBuilder().putArray("discovery.zen.ping.unicast.hosts",
-                addressA.address().getAddress().getHostAddress() + ":" + addressA.address().getPort(),
-                addressB.address().getAddress().getHostAddress() + ":" + addressB.address().getPort())
+                NetworkAddress.formatAddress(new InetSocketAddress(addressA.address().getAddress(), addressA.address().getPort())),
+                NetworkAddress.formatAddress(new InetSocketAddress(addressB.address().getAddress(), addressB.address().getPort())))
                 .build();
 
         UnicastZenPing zenPingA = new UnicastZenPing(hostsSettings, threadPool, transportServiceA, clusterName, Version.CURRENT, electMasterService, null);

core/src/test/java/org/elasticsearch/gateway/ReplicaShardAllocatorTests.java

@@ -21,6 +21,7 @@ package org.elasticsearch.gateway;
 
 import com.carrotsearch.randomizedtesting.generators.RandomPicks;
 import org.elasticsearch.Version;
+import org.elasticsearch.cluster.ClusterInfo;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
@@ -302,7 +303,7 @@ public class ReplicaShardAllocatorTests extends ESAllocationTestCase {
                 .metaData(metaData)
                 .routingTable(routingTable)
                 .nodes(DiscoveryNodes.builder().put(node1).put(node2).put(node3)).build();
-        return new RoutingAllocation(deciders, new RoutingNodes(state, false), state.nodes(), null);
+        return new RoutingAllocation(deciders, new RoutingNodes(state, false), state.nodes(), ClusterInfo.EMPTY);
     }
 
     private RoutingAllocation onePrimaryOnNode1And1ReplicaRecovering(AllocationDeciders deciders) {
@@ -321,7 +322,7 @@ public class ReplicaShardAllocatorTests extends ESAllocationTestCase {
                 .metaData(metaData)
                 .routingTable(routingTable)
                 .nodes(DiscoveryNodes.builder().put(node1).put(node2).put(node3)).build();
-        return new RoutingAllocation(deciders, new RoutingNodes(state, false), state.nodes(), null);
+        return new RoutingAllocation(deciders, new RoutingNodes(state, false), state.nodes(), ClusterInfo.EMPTY);
     }
 
     class TestAllocator extends ReplicaShardAllocator {

core/src/test/java/org/elasticsearch/http/netty/pipelining/HttpPipeliningHandlerTest.java

@@ -18,6 +18,7 @@
  */
 package org.elasticsearch.http.netty.pipelining;
 
+import org.elasticsearch.common.network.NetworkAddress;
 import org.elasticsearch.test.ESTestCase;
 import org.jboss.netty.bootstrap.ClientBootstrap;
 import org.jboss.netty.bootstrap.ServerBootstrap;
@@ -32,6 +33,7 @@ import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
 
+import java.net.InetAddress;
 import java.net.InetSocketAddress;
 import java.util.ArrayList;
 import java.util.List;
@@ -57,7 +59,7 @@ public class HttpPipeliningHandlerTest extends ESTestCase {
     private static final long CONNECTION_TIMEOUT = 10000L;
     private static final String CONTENT_TYPE_TEXT = "text/plain; charset=UTF-8";
     // TODO make me random
-    private static final InetSocketAddress HOST_ADDR = new InetSocketAddress("127.0.0.1", 9080);
+    private static final InetSocketAddress HOST_ADDR = new InetSocketAddress(InetAddress.getLoopbackAddress(), 9080);
     private static final String PATH1 = "/1";
     private static final String PATH2 = "/2";
     private static final String SOME_RESPONSE_TEXT = "some response for ";
@@ -123,13 +125,14 @@ public class HttpPipeliningHandlerTest extends ESTestCase {
         assertTrue(connectionFuture.await(CONNECTION_TIMEOUT));
         final Channel clientChannel = connectionFuture.getChannel();
 
+        // NetworkAddress.formatAddress makes a proper HOST header.
         final HttpRequest request1 = new DefaultHttpRequest(
                 HTTP_1_1, HttpMethod.GET, PATH1);
-        request1.headers().add(HOST, HOST_ADDR.toString());
+        request1.headers().add(HOST, NetworkAddress.formatAddress(HOST_ADDR));
 
         final HttpRequest request2 = new DefaultHttpRequest(
                 HTTP_1_1, HttpMethod.GET, PATH2);
-        request2.headers().add(HOST, HOST_ADDR.toString());
+        request2.headers().add(HOST, NetworkAddress.formatAddress(HOST_ADDR));
 
         clientChannel.write(request1);
         clientChannel.write(request2);

core/src/test/java/org/elasticsearch/index/TransportIndexFailuresIT.java

@@ -48,6 +48,7 @@ import static org.hamcrest.Matchers.equalTo;
  * Test failure when index replication actions fail mid-flight
  */
 @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, transportClientRatio = 0)
+@ESIntegTestCase.SuppressLocalMode
 public class TransportIndexFailuresIT extends ESIntegTestCase {
 
     private static final Settings nodeSettings = Settings.settingsBuilder()

core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java

@@ -25,6 +25,7 @@ import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.indices.stats.IndexStats;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.cluster.*;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardRoutingState;
@@ -456,6 +457,22 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         assertPathHasBeenCleared(idxPath);
     }
 
+    public void testExpectedShardSizeIsPresent() throws InterruptedException {
+        assertAcked(client().admin().indices().prepareCreate("test")
+                .setSettings(SETTING_NUMBER_OF_SHARDS, 1, SETTING_NUMBER_OF_REPLICAS, 0));
+        for (int i = 0; i < 50; i++) {
+            client().prepareIndex("test", "test").setSource("{}").get();
+        }
+        ensureGreen("test");
+        InternalClusterInfoService clusterInfoService = (InternalClusterInfoService) getInstanceFromNode(ClusterInfoService.class);
+        InternalClusterInfoService.ClusterInfoUpdateJob job = clusterInfoService.new ClusterInfoUpdateJob(false);
+        job.run();
+        ClusterState state = getInstanceFromNode(ClusterService.class).state();
+        Long test = clusterInfoService.getClusterInfo().getShardSize(state.getRoutingTable().index("test").getShards().get(0).primaryShard());
+        assertNotNull(test);
+        assertTrue(test > 0);
+    }
+
     public void testIndexCanChangeCustomDataPath() throws Exception {
         Environment env = getInstanceFromNode(Environment.class);
         Path idxPath = env.sharedDataFile().resolve(randomAsciiOfLength(10));

core/src/test/java/org/elasticsearch/indices/state/RareClusterStateIT.java

@@ -64,6 +64,7 @@ import static org.hamcrest.Matchers.*;
 /**
  */
 @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, numClientNodes = 0, transportClientRatio = 0)
+@ESIntegTestCase.SuppressLocalMode
 public class RareClusterStateIT extends ESIntegTestCase {
 
     @Override

core/src/test/java/org/elasticsearch/plugins/PluggableTransportModuleIT.java

@@ -23,6 +23,7 @@ import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.discovery.DiscoveryModule;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.transport.AssertingLocalTransport;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -49,6 +50,7 @@ public class PluggableTransportModuleIT extends ESIntegTestCase {
     protected Settings nodeSettings(int nodeOrdinal) {
         return settingsBuilder()
                 .put(super.nodeSettings(nodeOrdinal))
+                .put(DiscoveryModule.DISCOVERY_TYPE_KEY, "local")
                 .put("plugin.types", CountingSentRequestsPlugin.class.getName())
                 .build();
     }

core/src/test/java/org/elasticsearch/plugins/PluginManagerIT.java

@@ -20,6 +20,7 @@ package org.elasticsearch.plugins;
 
 import com.google.common.base.Charsets;
 import com.google.common.hash.Hashing;
+
 import org.apache.http.impl.client.HttpClients;
 import org.apache.lucene.util.LuceneTestCase;
 import org.elasticsearch.Version;
@@ -51,8 +52,10 @@ import org.junit.Test;
 import javax.net.ssl.HttpsURLConnection;
 import javax.net.ssl.SSLContext;
 import javax.net.ssl.SSLSocketFactory;
+
 import java.io.BufferedWriter;
 import java.io.IOException;
+import java.net.InetAddress;
 import java.net.InetSocketAddress;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.FileVisitResult;
@@ -605,7 +608,7 @@ public class PluginManagerIT extends ESIntegTestCase {
                 }
             });
 
-            Channel channel = serverBootstrap.bind(new InetSocketAddress("localhost", 0));
+            Channel channel = serverBootstrap.bind(new InetSocketAddress(InetAddress.getByName("localhost"), 0));
             int port = ((InetSocketAddress) channel.getLocalAddress()).getPort();
             // IO_ERROR because there is no real file delivered...
             assertStatus(String.format(Locale.ROOT, "install https://user:pass@localhost:%s/foo.zip --verbose --timeout 1s", port), ExitStatus.IO_ERROR);
@@ -645,7 +648,6 @@ public class PluginManagerIT extends ESIntegTestCase {
 
     private Tuple<Settings, Environment> buildInitialSettings() throws IOException {
         Settings settings = settingsBuilder()
-                .put("discovery.zen.ping.multicast.enabled", false)
                 .put("http.enabled", true)
                 .put("path.home", createTempDir()).build();
         return InternalSettingsPreparer.prepareSettings(settings, false);

core/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java

@@ -63,8 +63,10 @@ import org.elasticsearch.rest.RestResponse;
 import org.elasticsearch.rest.action.admin.cluster.repositories.get.RestGetRepositoriesAction;
 import org.elasticsearch.rest.action.admin.cluster.state.RestClusterStateAction;
 import org.elasticsearch.snapshots.mockstore.MockRepository;
+import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.InternalTestCluster;
 import org.elasticsearch.test.rest.FakeRestRequest;
+import org.elasticsearch.transport.TransportModule;
 import org.junit.Test;
 
 import java.io.IOException;
@@ -96,7 +98,8 @@ import static org.hamcrest.Matchers.nullValue;
 
 /**
  */
-@ClusterScope(scope = Scope.TEST, numDataNodes = 0)
+@ClusterScope(scope = Scope.TEST, numDataNodes = 0, transportClientRatio = 0)
+@ESIntegTestCase.SuppressLocalMode // TODO only restorePersistentSettingsTest needs this maybe factor out?
 public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCase {
 
     @Test

core/src/test/java/org/elasticsearch/stresstest/client/ClientFailover.java

@@ -25,6 +25,7 @@ import org.elasticsearch.common.transport.InetSocketTransportAddress;
 import org.elasticsearch.node.Node;
 import org.elasticsearch.node.NodeBuilder;
 
+import java.net.InetAddress;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicLong;
@@ -42,9 +43,9 @@ public class ClientFailover {
         // TODO: what is this? a public static void main test?!?!
 
         final TransportClient client = TransportClient.builder().build()
-                .addTransportAddress(new InetSocketTransportAddress("localhost", 9300))
-                .addTransportAddress(new InetSocketTransportAddress("localhost", 9301))
-                .addTransportAddress(new InetSocketTransportAddress("localhost", 9302));
+                .addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName("localhost"), 9300))
+                .addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName("localhost"), 9301))
+                .addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName("localhost"), 9302));
 
         final AtomicBoolean done = new AtomicBoolean();
         final AtomicLong indexed = new AtomicLong();

core/src/test/java/org/elasticsearch/stresstest/manyindices/ManyIndicesRemoteStressTest.java

@@ -28,6 +28,7 @@ import org.elasticsearch.common.transport.InetSocketTransportAddress;
 import org.elasticsearch.node.Node;
 import org.elasticsearch.node.NodeBuilder;
 
+import java.net.InetAddress;
 import java.util.Date;
 
 /**
@@ -49,7 +50,7 @@ public class ManyIndicesRemoteStressTest {
         Node node = null;
         // TODO: what is this? a public static void main test?!?!?!
         if (true) {
-            client = TransportClient.builder().settings(Settings.EMPTY).build().addTransportAddress(new InetSocketTransportAddress("localhost", 9300));
+            client = TransportClient.builder().settings(Settings.EMPTY).build().addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName("localhost"), 9300));
         } else {
             node = NodeBuilder.nodeBuilder().client(true).node();
             client = node.client();

core/src/test/java/org/elasticsearch/test/ESAllocationTestCase.java

@@ -19,6 +19,7 @@
 package org.elasticsearch.test;
 
 import org.elasticsearch.Version;
+import org.elasticsearch.cluster.ClusterInfoService;
 import org.elasticsearch.cluster.ClusterModule;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.EmptyClusterInfoService;
@@ -63,7 +64,7 @@ public abstract class ESAllocationTestCase extends ESTestCase {
     }
 
     public static AllocationService createAllocationService(Settings settings, Random random) {
-        return createAllocationService(settings,  new NodeSettingsService(Settings.Builder.EMPTY_SETTINGS), random);
+        return createAllocationService(settings, new NodeSettingsService(Settings.Builder.EMPTY_SETTINGS), random);
     }
 
     public static AllocationService createAllocationService(Settings settings, NodeSettingsService nodeSettingsService, Random random) {
@@ -72,6 +73,13 @@ public abstract class ESAllocationTestCase extends ESTestCase {
                 new ShardsAllocators(settings, NoopGatewayAllocator.INSTANCE), EmptyClusterInfoService.INSTANCE);
     }
 
+    public static AllocationService createAllocationService(Settings settings, ClusterInfoService clusterInfoService) {
+        return new AllocationService(settings,
+                randomAllocationDeciders(settings, new NodeSettingsService(Settings.Builder.EMPTY_SETTINGS), getRandom()),
+                new ShardsAllocators(settings, NoopGatewayAllocator.INSTANCE), clusterInfoService);
+    }
+
+
 
     public static AllocationDeciders randomAllocationDeciders(Settings settings, NodeSettingsService nodeSettingsService, Random random) {
         final List<Class<? extends AllocationDecider>> defaultAllocationDeciders = ClusterModule.DEFAULT_ALLOCATION_DECIDERS;

core/src/test/java/org/elasticsearch/test/ESIntegTestCase.java

@@ -32,6 +32,7 @@ import org.apache.http.impl.client.HttpClients;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.routing.UnassignedInfo;
 import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider;
+import org.elasticsearch.common.network.NetworkAddress;
 import org.elasticsearch.index.shard.MergeSchedulerConfig;
 import org.apache.lucene.util.IOUtils;
 import org.apache.lucene.util.LuceneTestCase;
@@ -117,6 +118,7 @@ import org.elasticsearch.search.SearchService;
 import org.elasticsearch.test.client.RandomizingClient;
 import org.elasticsearch.test.disruption.ServiceDisruptionScheme;
 import org.elasticsearch.test.rest.client.http.HttpRequestBuilder;
+import org.elasticsearch.transport.TransportModule;
 import org.hamcrest.Matchers;
 import org.joda.time.DateTimeZone;
 import org.junit.*;
@@ -124,7 +126,9 @@ import org.junit.*;
 import java.io.IOException;
 import java.io.InputStream;
 import java.lang.annotation.*;
+import java.net.InetAddress;
 import java.net.InetSocketAddress;
+import java.net.UnknownHostException;
 import java.nio.file.DirectoryStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -1042,40 +1046,34 @@ public abstract class ESIntegTestCase extends ESTestCase {
      */
     protected void ensureClusterStateConsistency() throws IOException {
         if (cluster() != null) {
-            boolean getResolvedAddress = InetSocketTransportAddress.getResolveAddress();
-            try {
-                InetSocketTransportAddress.setResolveAddress(false);
-                ClusterState masterClusterState = client().admin().cluster().prepareState().all().get().getState();
-                byte[] masterClusterStateBytes = ClusterState.Builder.toBytes(masterClusterState);
+            ClusterState masterClusterState = client().admin().cluster().prepareState().all().get().getState();
+            byte[] masterClusterStateBytes = ClusterState.Builder.toBytes(masterClusterState);
+            // remove local node reference
+            masterClusterState = ClusterState.Builder.fromBytes(masterClusterStateBytes, null);
+            Map<String, Object> masterStateMap = convertToMap(masterClusterState);
+            int masterClusterStateSize = masterClusterState.toString().length();
+            String masterId = masterClusterState.nodes().masterNodeId();
+            for (Client client : cluster()) {
+                ClusterState localClusterState = client.admin().cluster().prepareState().all().setLocal(true).get().getState();
+                byte[] localClusterStateBytes = ClusterState.Builder.toBytes(localClusterState);
                 // remove local node reference
-                masterClusterState = ClusterState.Builder.fromBytes(masterClusterStateBytes, null);
-                Map<String, Object> masterStateMap = convertToMap(masterClusterState);
-                int masterClusterStateSize = masterClusterState.toString().length();
-                String masterId = masterClusterState.nodes().masterNodeId();
-                for (Client client : cluster()) {
-                    ClusterState localClusterState = client.admin().cluster().prepareState().all().setLocal(true).get().getState();
-                    byte[] localClusterStateBytes = ClusterState.Builder.toBytes(localClusterState);
-                    // remove local node reference
-                    localClusterState = ClusterState.Builder.fromBytes(localClusterStateBytes, null);
-                    final Map<String, Object> localStateMap = convertToMap(localClusterState);
-                    final int localClusterStateSize = localClusterState.toString().length();
-                    // Check that the non-master node has the same version of the cluster state as the master and that this node didn't disconnect from the master
-                    if (masterClusterState.version() == localClusterState.version() && localClusterState.nodes().nodes().containsKey(masterId)) {
-                        try {
-                            assertEquals("clusterstate UUID does not match", masterClusterState.stateUUID(), localClusterState.stateUUID());
-                            // We cannot compare serialization bytes since serialization order of maps is not guaranteed
-                            // but we can compare serialization sizes - they should be the same
-                            assertEquals("clusterstate size does not match", masterClusterStateSize, localClusterStateSize);
-                            // Compare JSON serialization
-                            assertNull("clusterstate JSON serialization does not match", differenceBetweenMapsIgnoringArrayOrder(masterStateMap, localStateMap));
-                        } catch (AssertionError error) {
-                            logger.error("Cluster state from master:\n{}\nLocal cluster state:\n{}", masterClusterState.toString(), localClusterState.toString());
-                            throw error;
-                        }
+                localClusterState = ClusterState.Builder.fromBytes(localClusterStateBytes, null);
+                final Map<String, Object> localStateMap = convertToMap(localClusterState);
+                final int localClusterStateSize = localClusterState.toString().length();
+                // Check that the non-master node has the same version of the cluster state as the master and that this node didn't disconnect from the master
+                if (masterClusterState.version() == localClusterState.version() && localClusterState.nodes().nodes().containsKey(masterId)) {
+                    try {
+                        assertEquals("clusterstate UUID does not match", masterClusterState.stateUUID(), localClusterState.stateUUID());
+                        // We cannot compare serialization bytes since serialization order of maps is not guaranteed
+                        // but we can compare serialization sizes - they should be the same
+                        assertEquals("clusterstate size does not match", masterClusterStateSize, localClusterStateSize);
+                        // Compare JSON serialization
+                        assertNull("clusterstate JSON serialization does not match", differenceBetweenMapsIgnoringArrayOrder(masterStateMap, localStateMap));
+                    } catch (AssertionError error) {
+                        logger.error("Cluster state from master:\n{}\nLocal cluster state:\n{}", masterClusterState.toString(), localClusterState.toString());
+                        throw error;
                     }
                 }
-            } finally {
-                InetSocketTransportAddress.setResolveAddress(getResolvedAddress);
             }
         }
 
@@ -1553,49 +1551,51 @@ public abstract class ESIntegTestCase extends ESTestCase {
         assertThat(clearResponse.isSucceeded(), equalTo(true));
     }
 
-    private static ClusterScope getAnnotation(Class<?> clazz) {
+    private static <A extends Annotation> A getAnnotation(Class<?> clazz, Class<A> annotationClass) {
         if (clazz == Object.class || clazz == ESIntegTestCase.class) {
             return null;
         }
-        ClusterScope annotation = clazz.getAnnotation(ClusterScope.class);
+        A annotation = clazz.getAnnotation(annotationClass);
         if (annotation != null) {
             return annotation;
         }
-        return getAnnotation(clazz.getSuperclass());
+        return getAnnotation(clazz.getSuperclass(), annotationClass);
     }
 
+
+
     private Scope getCurrentClusterScope() {
         return getCurrentClusterScope(this.getClass());
     }
 
     private static Scope getCurrentClusterScope(Class<?> clazz) {
-        ClusterScope annotation = getAnnotation(clazz);
+        ClusterScope annotation = getAnnotation(clazz, ClusterScope.class);
         // if we are not annotated assume suite!
         return annotation == null ? Scope.SUITE : annotation.scope();
     }
 
     private int getNumDataNodes() {
-        ClusterScope annotation = getAnnotation(this.getClass());
+        ClusterScope annotation = getAnnotation(this.getClass(), ClusterScope.class);
         return annotation == null ? -1 : annotation.numDataNodes();
     }
 
     private int getMinNumDataNodes() {
-        ClusterScope annotation = getAnnotation(this.getClass());
+        ClusterScope annotation = getAnnotation(this.getClass(), ClusterScope.class);
         return annotation == null || annotation.minNumDataNodes() == -1 ? InternalTestCluster.DEFAULT_MIN_NUM_DATA_NODES : annotation.minNumDataNodes();
     }
 
     private int getMaxNumDataNodes() {
-        ClusterScope annotation = getAnnotation(this.getClass());
+        ClusterScope annotation = getAnnotation(this.getClass(), ClusterScope.class);
         return annotation == null || annotation.maxNumDataNodes() == -1 ? InternalTestCluster.DEFAULT_MAX_NUM_DATA_NODES : annotation.maxNumDataNodes();
     }
 
     private int getNumClientNodes() {
-        ClusterScope annotation = getAnnotation(this.getClass());
+        ClusterScope annotation = getAnnotation(this.getClass(), ClusterScope.class);
         return annotation == null ? InternalTestCluster.DEFAULT_NUM_CLIENT_NODES : annotation.numClientNodes();
     }
 
     private boolean randomDynamicTemplates() {
-        ClusterScope annotation = getAnnotation(this.getClass());
+        ClusterScope annotation = getAnnotation(this.getClass(), ClusterScope.class);
         return annotation == null || annotation.randomDynamicTemplates();
     }
 
@@ -1607,7 +1607,7 @@ public abstract class ESIntegTestCase extends ESTestCase {
      * In other words subclasses must ensure this method is idempotent.
      */
     protected Settings nodeSettings(int nodeOrdinal) {
-        return settingsBuilder()
+        Settings.Builder builder = settingsBuilder()
                 // Default the watermarks to absurdly low to prevent the tests
                 // from failing on nodes without enough disk space
                 .put(DiskThresholdDecider.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK, "1b")
@@ -1615,8 +1615,8 @@ public abstract class ESIntegTestCase extends ESTestCase {
                 .put("script.indexed", "on")
                 .put("script.inline", "on")
                         // wait short time for other active shards before actually deleting, default 30s not needed in tests
-                .put(IndicesStore.INDICES_STORE_DELETE_SHARD_TIMEOUT, new TimeValue(1, TimeUnit.SECONDS))
-                .build();
+                .put(IndicesStore.INDICES_STORE_DELETE_SHARD_TIMEOUT, new TimeValue(1, TimeUnit.SECONDS));
+        return builder.build();
     }
 
     /**
@@ -1629,7 +1629,7 @@ public abstract class ESIntegTestCase extends ESTestCase {
         return Settings.EMPTY;
     }
 
-    private ExternalTestCluster buildExternalCluster(String clusterAddresses) {
+    private ExternalTestCluster buildExternalCluster(String clusterAddresses) throws UnknownHostException {
         String[] stringAddresses = clusterAddresses.split(",");
         TransportAddress[] transportAddresses = new TransportAddress[stringAddresses.length];
         int i = 0;
@@ -1639,7 +1639,7 @@ public abstract class ESIntegTestCase extends ESTestCase {
                 throw new IllegalArgumentException("address [" + clusterAddresses + "] not valid");
             }
             try {
-                transportAddresses[i++] = new InetSocketTransportAddress(split[0], Integer.valueOf(split[1]));
+                transportAddresses[i++] = new InetSocketTransportAddress(InetAddress.getByName(split[0]), Integer.valueOf(split[1]));
             } catch (NumberFormatException e) {
                 throw new IllegalArgumentException("port is not valid, expected number but was [" + split[1] + "]");
             }
@@ -1693,7 +1693,18 @@ public abstract class ESIntegTestCase extends ESTestCase {
             minNumDataNodes = getMinNumDataNodes();
             maxNumDataNodes = getMaxNumDataNodes();
         }
-        return new InternalTestCluster(seed, createTempDir(), minNumDataNodes, maxNumDataNodes,
+        SuppressLocalMode noLocal = getAnnotation(this.getClass(), SuppressLocalMode.class);
+        SuppressNetworkMode noNetwork = getAnnotation(this.getClass(), SuppressNetworkMode.class);
+        String nodeMode = InternalTestCluster.configuredNodeMode();
+        if (noLocal != null && noNetwork != null) {
+            throw new IllegalStateException("Can't suppress both network and local mode");
+        } else if (noLocal != null){
+            nodeMode = "network";
+        } else if (noNetwork != null) {
+            nodeMode = "local";
+        }
+
+        return new InternalTestCluster(nodeMode, seed, createTempDir(), minNumDataNodes, maxNumDataNodes,
                 InternalTestCluster.clusterName(scope.name(), seed) + "-cluster", settingsSource, getNumClientNodes(),
                 InternalTestCluster.DEFAULT_ENABLE_HTTP_PIPELINING, nodePrefix);
     }
@@ -1715,7 +1726,7 @@ public abstract class ESIntegTestCase extends ESTestCase {
      * return a random ratio in the interval <tt>[0..1]</tt>
      */
     protected double getPerTestTransportClientRatio() {
-        final ClusterScope annotation = getAnnotation(this.getClass());
+        final ClusterScope annotation = getAnnotation(this.getClass(), ClusterScope.class);
         double perTestRatio = -1;
         if (annotation != null) {
             perTestRatio = annotation.transportClientRatio();
@@ -1947,7 +1958,7 @@ public abstract class ESIntegTestCase extends ESTestCase {
         TransportAddress publishAddress = randomFrom(nodes).getHttp().address().publishAddress();
         assertEquals(1, publishAddress.uniqueAddressTypeId());
         InetSocketAddress address = ((InetSocketTransportAddress) publishAddress).address();
-        return new HttpRequestBuilder(HttpClients.createDefault()).host(address.getHostName()).port(address.getPort());
+        return new HttpRequestBuilder(HttpClients.createDefault()).host(NetworkAddress.formatAddress(address.getAddress())).port(address.getPort());
     }
 
     /**
@@ -1973,4 +1984,19 @@ public abstract class ESIntegTestCase extends ESTestCase {
     @Inherited
     public @interface SuiteScopeTestCase {
     }
+
+    /**
+     * If used the test will never run in local mode.
+     */
+    @Retention(RetentionPolicy.RUNTIME)
+    @Inherited
+    public @interface SuppressLocalMode {}
+
+    /**
+     * If used the test will never run in network mode
+     */
+    @Retention(RetentionPolicy.RUNTIME)
+    @Inherited
+    public @interface SuppressNetworkMode {}
+
 }

core/src/test/java/org/elasticsearch/test/InternalTestCluster.java

@@ -192,8 +192,6 @@ public final class InternalTestCluster extends TestCluster {
 
     static final boolean DEFAULT_ENABLE_HTTP_PIPELINING = true;
 
-    public static final String NODE_MODE = nodeMode();
-
     /* sorted map to make traverse order reproducible, concurrent since we do checks on it not within a sync block */
     private final NavigableMap<String, NodeAndClient> nodes = new TreeMap<>();
 
@@ -227,16 +225,21 @@ public final class InternalTestCluster extends TestCluster {
     private final Path baseDir;
 
     private ServiceDisruptionScheme activeDisruptionScheme;
+    private String nodeMode;
 
-    public InternalTestCluster(long clusterSeed, Path baseDir, int minNumDataNodes, int maxNumDataNodes, String clusterName, int numClientNodes,
+    public InternalTestCluster(String nodeMode, long clusterSeed, Path baseDir, int minNumDataNodes, int maxNumDataNodes, String clusterName, int numClientNodes,
                                boolean enableHttpPipelining, String nodePrefix) {
-        this(clusterSeed, baseDir, minNumDataNodes, maxNumDataNodes, clusterName, DEFAULT_SETTINGS_SOURCE, numClientNodes, enableHttpPipelining, nodePrefix);
+        this(nodeMode, clusterSeed, baseDir, minNumDataNodes, maxNumDataNodes, clusterName, DEFAULT_SETTINGS_SOURCE, numClientNodes, enableHttpPipelining, nodePrefix);
     }
 
-    public InternalTestCluster(long clusterSeed, Path baseDir,
+    public InternalTestCluster(String nodeMode, long clusterSeed, Path baseDir,
                                int minNumDataNodes, int maxNumDataNodes, String clusterName, SettingsSource settingsSource, int numClientNodes,
                                boolean enableHttpPipelining, String nodePrefix) {
         super(clusterSeed);
+        if ("network".equals(nodeMode) == false && "local".equals(nodeMode) == false) {
+            throw new IllegalArgumentException("Unknown nodeMode: " + nodeMode);
+        }
+        this.nodeMode = nodeMode;
         this.baseDir = baseDir;
         this.clusterName = clusterName;
         if (minNumDataNodes < 0 || maxNumDataNodes < 0) {
@@ -300,7 +303,7 @@ public final class InternalTestCluster extends TestCluster {
         builder.put("transport.tcp.port", BASE_PORT + "-" + (BASE_PORT + 100));
         builder.put("http.port", BASE_PORT + 101 + "-" + (BASE_PORT + 200));
         builder.put(InternalSettingsPreparer.IGNORE_SYSTEM_PROPERTIES_SETTING, true);
-        builder.put("node.mode", NODE_MODE);
+        builder.put("node.mode", nodeMode);
         builder.put("http.pipelining", enableHttpPipelining);
         if (Strings.hasLength(System.getProperty("es.logger.level"))) {
             builder.put("logger.level", System.getProperty("es.logger.level"));
@@ -327,7 +330,7 @@ public final class InternalTestCluster extends TestCluster {
         executor = EsExecutors.newCached("test runner", 0, TimeUnit.SECONDS, EsExecutors.daemonThreadFactory("test_" + clusterName));
     }
 
-    public static String nodeMode() {
+    public static String configuredNodeMode() {
         Builder builder = Settings.builder();
         if (Strings.isEmpty(System.getProperty("es.node.mode")) && Strings.isEmpty(System.getProperty("es.node.local"))) {
             return "local"; // default if nothing is specified
@@ -354,11 +357,8 @@ public final class InternalTestCluster extends TestCluster {
         return nodes.keySet().toArray(Strings.EMPTY_ARRAY);
     }
 
-    private static boolean isLocalTransportConfigured() {
-        if ("local".equals(System.getProperty("es.node.mode", "network"))) {
-            return true;
-        }
-        return Boolean.parseBoolean(System.getProperty("es.node.local", "false"));
+    private boolean isLocalTransportConfigured() {
+        return "local".equals(nodeMode);
     }
 
     private Settings getSettings(int nodeOrdinal, long nodeSeed, Settings others) {
@@ -378,7 +378,7 @@ public final class InternalTestCluster extends TestCluster {
         return builder.build();
     }
 
-    private static Settings getRandomNodeSettings(long seed) {
+    private Settings getRandomNodeSettings(long seed) {
         Random random = new Random(seed);
         Builder builder = Settings.settingsBuilder()
                 .put(SETTING_CLUSTER_NODE_SEED, seed);
@@ -782,6 +782,10 @@ public final class InternalTestCluster extends TestCluster {
         }
     }
 
+    public String getNodeMode() {
+        return nodeMode;
+    }
+
     private final class NodeAndClient implements Closeable {
         private Node node;
         private Client nodeClient;
@@ -844,7 +848,7 @@ public final class InternalTestCluster extends TestCluster {
             /* no sniff client for now - doesn't work will all tests since it might throw NoNodeAvailableException if nodes are shut down.
              * we first need support of transportClientRatio as annotations or so
              */
-            return transportClient = new TransportClientFactory(false, settingsSource.transportClient(), baseDir).client(node, clusterName);
+            return transportClient = new TransportClientFactory(false, settingsSource.transportClient(), baseDir, nodeMode).client(node, clusterName);
         }
 
         void resetClient() throws IOException {
@@ -901,11 +905,13 @@ public final class InternalTestCluster extends TestCluster {
         private final boolean sniff;
         private final Settings settings;
         private final Path baseDir;
+        private final String nodeMode;
 
-        TransportClientFactory(boolean sniff, Settings settings, Path baseDir) {
+        TransportClientFactory(boolean sniff, Settings settings, Path baseDir, String nodeMode) {
             this.sniff = sniff;
             this.settings = settings != null ? settings : Settings.EMPTY;
             this.baseDir = baseDir;
+            this.nodeMode = nodeMode;
         }
 
         public Client client(Node node, String clusterName) {
@@ -916,7 +922,7 @@ public final class InternalTestCluster extends TestCluster {
                     .put("path.home", baseDir)
                     .put("name", TRANSPORT_CLIENT_PREFIX + node.settings().get("name"))
                     .put(ClusterName.SETTING, clusterName).put("client.transport.sniff", sniff)
-                    .put("node.mode", nodeSettings.get("node.mode", NODE_MODE))
+                    .put("node.mode", nodeSettings.get("node.mode", nodeMode))
                     .put("node.local", nodeSettings.get("node.local", ""))
                     .put("logger.prefix", nodeSettings.get("logger.prefix", ""))
                     .put("logger.level", nodeSettings.get("logger.level", "INFO"))

core/src/test/java/org/elasticsearch/test/TestCluster.java

@@ -211,4 +211,6 @@ public abstract class TestCluster implements Iterable<Client>, Closeable {
      * Returns the cluster name
      */
     public abstract String getClusterName();
+
+
 }

core/src/test/java/org/elasticsearch/test/discovery/ClusterDiscoveryConfiguration.java

@@ -21,6 +21,7 @@ package org.elasticsearch.test.discovery;
 import com.carrotsearch.randomizedtesting.RandomizedTest;
 import com.google.common.primitives.Ints;
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.common.network.NetworkUtils;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.InternalTestCluster;
@@ -28,15 +29,17 @@ import org.elasticsearch.test.SettingsSource;
 import org.elasticsearch.transport.local.LocalTransport;
 
 import java.io.IOException;
+import java.net.Inet4Address;
+import java.net.InetAddress;
 import java.net.InetSocketAddress;
 import java.net.ServerSocket;
-import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.Set;
 
 public class ClusterDiscoveryConfiguration extends SettingsSource {
 
     static Settings DEFAULT_NODE_SETTINGS = Settings.settingsBuilder().put("discovery.type", "zen").build();
+    private static final String IP_ADDR = "127.0.0.1";
 
     final int numOfNodes;
     final Settings nodeSettings;
@@ -63,29 +66,15 @@ public class ClusterDiscoveryConfiguration extends SettingsSource {
         // this variable is incremented on each bind attempt and will maintain the next port that should be tried
         private static int nextPort = calcBasePort();
 
-        // since we run multiple test iterations, we need some flexibility in the choice of ports a node can have (port may
-        // stay in use by previous iterations on some OSes - read CentOs). This controls the amount of ports each node
-        // is assigned. All ports in range will be added to the unicast hosts, which is OK because we know only one will be used.
-        private static final int NUM_PORTS_PER_NODE = 3;
-
-        private final String[] unicastHosts;
-        private final boolean localMode;
-
-        public UnicastZen(int numOfNodes) {
-            this(numOfNodes, numOfNodes);
-        }
+        private final int[] unicastHostOrdinals;
+        private final int[] unicastHostPorts;
 
         public UnicastZen(int numOfNodes, Settings extraSettings) {
             this(numOfNodes, numOfNodes, extraSettings);
         }
 
-        public UnicastZen(int numOfNodes, int numOfUnicastHosts) {
-            this(numOfNodes, numOfUnicastHosts, Settings.EMPTY);
-        }
-
         public UnicastZen(int numOfNodes, int numOfUnicastHosts, Settings extraSettings) {
             super(numOfNodes, extraSettings);
-            int[] unicastHostOrdinals;
             if (numOfUnicastHosts == numOfNodes) {
                 unicastHostOrdinals = new int[numOfNodes];
                 for (int i = 0; i < numOfNodes; i++) {
@@ -98,8 +87,8 @@ public class ClusterDiscoveryConfiguration extends SettingsSource {
                 }
                 unicastHostOrdinals = Ints.toArray(ordinals);
             }
-            this.localMode = nodeSettings.get("node.mode", InternalTestCluster.NODE_MODE).equals("local");
-            this.unicastHosts = buildUnicastHostSetting(unicastHostOrdinals, localMode);
+            this.unicastHostPorts = unicastHostPorts(numOfNodes);
+            assert unicastHostOrdinals.length <= unicastHostPorts.length;
         }
 
         public UnicastZen(int numOfNodes, int[] unicastHostOrdinals) {
@@ -108,59 +97,70 @@ public class ClusterDiscoveryConfiguration extends SettingsSource {
 
         public UnicastZen(int numOfNodes, Settings extraSettings, int[] unicastHostOrdinals) {
             super(numOfNodes, extraSettings);
-            this.localMode = nodeSettings.get("node.mode", InternalTestCluster.NODE_MODE).equals("local");
-            this.unicastHosts = buildUnicastHostSetting(unicastHostOrdinals, localMode);
+            this.unicastHostOrdinals = unicastHostOrdinals;
+            this.unicastHostPorts = unicastHostPorts(numOfNodes);
+            assert unicastHostOrdinals.length <= unicastHostPorts.length;
         }
 
         private static int calcBasePort() {
             return 30000 + InternalTestCluster.BASE_PORT;
         }
 
-        private static String[] buildUnicastHostSetting(int[] unicastHostOrdinals, boolean localMode) {
-            ArrayList<String> unicastHosts = new ArrayList<>();
-            for (int i = 0; i < unicastHostOrdinals.length; i++) {
-                final int hostOrdinal = unicastHostOrdinals[i];
-                if (localMode) {
-                    unicastHosts.add("node_" + hostOrdinal);
-                } else {
-                    // we need to pin the node port & host so we'd know where to point things
-                    final int[] ports = nodePorts(hostOrdinal);
-                    for (int port : ports) {
-                        unicastHosts.add("localhost:" + port);
-                    }
-                }
-            }
-            return unicastHosts.toArray(new String[unicastHosts.size()]);
-        }
-
         @Override
         public Settings node(int nodeOrdinal) {
-            Settings.Builder builder = Settings.builder()
-                    .put("discovery.zen.ping.multicast.enabled", false);
+            Settings.Builder builder = Settings.builder();
 
-            if (localMode) {
-                builder.put(LocalTransport.TRANSPORT_LOCAL_ADDRESS, "node_" + nodeOrdinal);
+            String[] unicastHosts = new String[unicastHostOrdinals.length];
+            if (nodeOrdinal >= unicastHostPorts.length) {
+                throw new ElasticsearchException("nodeOrdinal [" + nodeOrdinal + "] is greater than the number unicast ports [" + unicastHostPorts.length + "]");
             } else {
                 // we need to pin the node port & host so we'd know where to point things
-                String ports = "";
-                for (int port : nodePorts(nodeOrdinal)) {
-                    ports += "," + port;
+                builder.put("transport.tcp.port", unicastHostPorts[nodeOrdinal]);
+                builder.put("transport.host", IP_ADDR); // only bind on one IF we use v4 here by default
+                builder.put("transport.bind_host", IP_ADDR);
+                builder.put("transport.publish_host", IP_ADDR);
+                builder.put("http.enabled", false);
+                for (int i = 0; i < unicastHostOrdinals.length; i++) {
+                    unicastHosts[i] = IP_ADDR + ":" + (unicastHostPorts[unicastHostOrdinals[i]]);
                 }
-                builder.put("transport.tcp.port", ports.substring(1));
-                builder.put("transport.host", "localhost");
             }
             builder.putArray("discovery.zen.ping.unicast.hosts", unicastHosts);
             return builder.put(super.node(nodeOrdinal)).build();
         }
 
-        protected static int[] nodePorts(int nodeOridnal) {
-            int[] unicastHostPorts = new int[NUM_PORTS_PER_NODE];
+        @SuppressForbidden(reason = "we know we pass a IP address")
+        protected synchronized static int[] unicastHostPorts(int numHosts) {
+            int[] unicastHostPorts = new int[numHosts];
 
-            final int basePort = calcBasePort() + nodeOridnal * NUM_PORTS_PER_NODE;
+            final int basePort = calcBasePort();
+            final int maxPort = basePort + InternalTestCluster.PORTS_PER_JVM;
+            int tries = 0;
             for (int i = 0; i < unicastHostPorts.length; i++) {
-                unicastHostPorts[i] = basePort + i;
-            }
+                boolean foundPortInRange = false;
+                while (tries < InternalTestCluster.PORTS_PER_JVM && !foundPortInRange) {
+                    try (ServerSocket serverSocket = new ServerSocket()) {
+                        // Set SO_REUSEADDR as we may bind here and not be able to reuse the address immediately without it.
+                        serverSocket.setReuseAddress(NetworkUtils.defaultReuseAddress());
+                        serverSocket.bind(new InetSocketAddress(IP_ADDR, nextPort));
+                        // bind was a success
+                        foundPortInRange = true;
+                        unicastHostPorts[i] = nextPort;
+                    } catch (IOException e) {
+                        // Do nothing
+                    }
 
+                    nextPort++;
+                    if (nextPort >= maxPort) {
+                        // Roll back to the beginning of the range and do not go into another JVM's port range
+                        nextPort = basePort;
+                    }
+                    tries++;
+                }
+
+                if (!foundPortInRange) {
+                    throw new ElasticsearchException("could not find enough open ports in range [" + basePort + "-" + maxPort + "]. required [" + unicastHostPorts.length + "] ports");
+                }
+            }
             return unicastHostPorts;
         }
     }

core/src/test/java/org/elasticsearch/test/rest/client/RestClient.java

@@ -39,6 +39,7 @@ import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.PathUtils;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.logging.Loggers;
+import org.elasticsearch.common.network.NetworkAddress;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.rest.client.http.HttpRequestBuilder;
 import org.elasticsearch.test.rest.client.http.HttpResponse;
@@ -247,7 +248,7 @@ public class RestClient implements Closeable {
         return new HttpRequestBuilder(httpClient)
                 .addHeaders(headers)
                 .protocol(protocol)
-                .host(address.getHostName()).port(address.getPort());
+                .host(NetworkAddress.formatAddress(address.getAddress())).port(address.getPort());
     }
 
     protected HttpRequestBuilder httpRequestBuilder() {

core/src/test/java/org/elasticsearch/test/rest/client/http/HttpRequestBuilder.java

@@ -27,6 +27,7 @@ import org.elasticsearch.client.support.Headers;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.logging.Loggers;
+import org.elasticsearch.common.network.NetworkAddress;
 import org.elasticsearch.common.transport.InetSocketTransportAddress;
 import org.elasticsearch.http.HttpServerTransport;
 
@@ -77,7 +78,7 @@ public class HttpRequestBuilder {
 
     public HttpRequestBuilder httpTransport(HttpServerTransport httpServerTransport) {
         InetSocketTransportAddress transportAddress = (InetSocketTransportAddress) httpServerTransport.boundAddress().publishAddress();
-        return host(transportAddress.address().getHostName()).port(transportAddress.address().getPort());
+        return host(NetworkAddress.formatAddress(transportAddress.address().getAddress())).port(transportAddress.address().getPort());
     }
 
     public HttpRequestBuilder port(int port) {

core/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java

@@ -55,8 +55,8 @@ public class InternalTestClusterTests extends ESTestCase {
         String nodePrefix = randomRealisticUnicodeOfCodepointLengthBetween(1, 10);
 
         Path baseDir = createTempDir();
-        InternalTestCluster cluster0 = new InternalTestCluster(clusterSeed, baseDir, minNumDataNodes, maxNumDataNodes, clusterName, settingsSource, numClientNodes, enableHttpPipelining, nodePrefix);
-        InternalTestCluster cluster1 = new InternalTestCluster(clusterSeed, baseDir, minNumDataNodes, maxNumDataNodes, clusterName, settingsSource, numClientNodes, enableHttpPipelining, nodePrefix);
+        InternalTestCluster cluster0 = new InternalTestCluster("local", clusterSeed, baseDir, minNumDataNodes, maxNumDataNodes, clusterName, settingsSource, numClientNodes, enableHttpPipelining, nodePrefix);
+        InternalTestCluster cluster1 = new InternalTestCluster("local", clusterSeed, baseDir, minNumDataNodes, maxNumDataNodes, clusterName, settingsSource, numClientNodes, enableHttpPipelining, nodePrefix);
         assertClusters(cluster0, cluster1, true);
 
     }
@@ -99,8 +99,8 @@ public class InternalTestClusterTests extends ESTestCase {
         String nodePrefix = "foobar";
 
         Path baseDir = createTempDir();
-        InternalTestCluster cluster0 = new InternalTestCluster(clusterSeed, baseDir, minNumDataNodes, maxNumDataNodes, clusterName1, settingsSource, numClientNodes, enableHttpPipelining, nodePrefix);
-        InternalTestCluster cluster1 = new InternalTestCluster(clusterSeed, baseDir, minNumDataNodes, maxNumDataNodes, clusterName2, settingsSource, numClientNodes, enableHttpPipelining, nodePrefix);
+        InternalTestCluster cluster0 = new InternalTestCluster("local", clusterSeed, baseDir, minNumDataNodes, maxNumDataNodes, clusterName1, settingsSource, numClientNodes, enableHttpPipelining, nodePrefix);
+        InternalTestCluster cluster1 = new InternalTestCluster("local", clusterSeed, baseDir, minNumDataNodes, maxNumDataNodes, clusterName2, settingsSource, numClientNodes, enableHttpPipelining, nodePrefix);
 
         assertClusters(cluster0, cluster1, false);
         long seed = randomLong();
@@ -124,7 +124,6 @@ public class InternalTestClusterTests extends ESTestCase {
             cluster0.afterTest();
             cluster1.afterTest();
         } finally {
-
             IOUtils.close(cluster0, cluster1);
         }
     }

core/src/test/java/org/elasticsearch/test/transport/CapturingTransport.java

@@ -28,10 +28,7 @@ import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.transport.*;
 
 import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
 import java.util.concurrent.BlockingQueue;
 
 /** A transport class that doesn't send anything but rather captures all requests for inspection from tests */
@@ -114,7 +111,8 @@ public class CapturingTransport implements Transport {
     }
 
     @Override
-    public TransportAddress[] addressesFromString(String address) throws Exception {
+    public TransportAddress[] addressesFromString(String address, int perAddressLimit) throws Exception {
+        // WTF
         return new TransportAddress[0];
     }
 
@@ -177,4 +175,9 @@ public class CapturingTransport implements Transport {
     public void close() {
 
     }
+
+    @Override
+    public List<String> getLocalAddresses() {
+        return Collections.EMPTY_LIST;
+    }
 }

core/src/test/java/org/elasticsearch/test/transport/MockTransportService.java

@@ -362,8 +362,8 @@ public class MockTransportService extends TransportService {
         }
 
         @Override
-        public TransportAddress[] addressesFromString(String address) throws Exception {
-            return transport.addressesFromString(address);
+        public TransportAddress[] addressesFromString(String address, int perAddressLimit) throws Exception {
+            return transport.addressesFromString(address, perAddressLimit);
         }
 
         @Override
@@ -401,6 +401,11 @@ public class MockTransportService extends TransportService {
             return transport.serverOpen();
         }
 
+        @Override
+        public List<String> getLocalAddresses() {
+            return transport.getLocalAddresses();
+        }
+
         @Override
         public Lifecycle.State lifecycleState() {
             return transport.lifecycleState();

core/src/test/java/org/elasticsearch/threadpool/SimpleThreadPoolIT.java

@@ -23,7 +23,6 @@ import com.google.common.collect.Sets;
 import org.elasticsearch.action.admin.cluster.node.info.NodeInfo;
 import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse;
 import org.elasticsearch.action.index.IndexRequestBuilder;
-import org.elasticsearch.common.network.MulticastChannel;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -112,8 +111,7 @@ public class SimpleThreadPoolIT extends ESIntegTestCase {
         for (String threadName : threadNames) {
             // ignore some shared threads we know that are created within the same VM, like the shared discovery one
             // or the ones that are occasionally come up from ESSingleNodeTestCase
-            if (threadName.contains("[" + MulticastChannel.SHARED_CHANNEL_NAME + "]")
-                    || threadName.contains("[" + ESSingleNodeTestCase.nodeName() + "]")
+            if (threadName.contains("[" + ESSingleNodeTestCase.nodeName() + "]")
                     || threadName.contains("Keep-Alive-Timer")) {
                 continue;
             }

core/src/test/java/org/elasticsearch/transport/NettySizeHeaderFrameDecoderTests.java

@@ -20,6 +20,7 @@
 package org.elasticsearch.transport;
 
 import com.google.common.base.Charsets;
+
 import org.elasticsearch.Version;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.network.NetworkService;
@@ -39,6 +40,7 @@ import org.junit.Test;
 
 import java.io.BufferedReader;
 import java.io.InputStreamReader;
+import java.net.InetAddress;
 import java.net.Socket;
 
 import static org.elasticsearch.common.settings.Settings.settingsBuilder;
@@ -55,7 +57,7 @@ public class NettySizeHeaderFrameDecoderTests extends ESTestCase {
     private ThreadPool threadPool;
     private NettyTransport nettyTransport;
     private int port;
-    private String host;
+    private InetAddress host;
 
     @Before
     public void startThreadPool() {
@@ -70,7 +72,7 @@ public class NettySizeHeaderFrameDecoderTests extends ESTestCase {
 
         InetSocketTransportAddress transportAddress = (InetSocketTransportAddress) nettyTransport.boundAddress().boundAddress();
         port = transportAddress.address().getPort();
-        host = transportAddress.address().getHostString();
+        host = transportAddress.address().getAddress();
 
     }
 

core/src/test/java/org/elasticsearch/transport/netty/NettyTransportMultiPortIntegrationIT.java

@@ -23,6 +23,7 @@ import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.action.admin.cluster.node.info.NodeInfo;
 import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse;
 import org.elasticsearch.client.transport.TransportClient;
+import org.elasticsearch.common.network.NetworkAddress;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.InetSocketTransportAddress;
 import org.elasticsearch.test.ESIntegTestCase;
@@ -32,6 +33,7 @@ import org.elasticsearch.test.transport.MockTransportService;
 import org.elasticsearch.transport.TransportModule;
 import org.junit.Test;
 
+import java.net.InetAddress;
 import java.util.Locale;
 
 import static org.elasticsearch.common.settings.Settings.settingsBuilder;
@@ -71,7 +73,7 @@ public class NettyTransportMultiPortIntegrationIT extends ESIntegTestCase {
                 .put("path.home", createTempDir().toString())
                 .build();
         try (TransportClient transportClient = TransportClient.builder().settings(settings).loadConfigSettings(false).build()) {
-            transportClient.addTransportAddress(new InetSocketTransportAddress("127.0.0.1", randomPort));
+            transportClient.addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName("127.0.0.1"), randomPort));
             ClusterHealthResponse response = transportClient.admin().cluster().prepareHealth().get();
             assertThat(response.getStatus(), is(ClusterHealthStatus.GREEN));
         }
@@ -93,7 +95,7 @@ public class NettyTransportMultiPortIntegrationIT extends ESIntegTestCase {
             // publish address
             assertThat(nodeInfo.getTransport().getProfileAddresses().get("client1").publishAddress(), instanceOf(InetSocketTransportAddress.class));
             InetSocketTransportAddress publishAddress = (InetSocketTransportAddress) nodeInfo.getTransport().getProfileAddresses().get("client1").publishAddress();
-            assertThat(publishAddress.address().getHostName(), is("127.0.0.7"));
+            assertThat(NetworkAddress.formatAddress(publishAddress.address().getAddress()), is("127.0.0.7"));
             assertThat(publishAddress.address().getPort(), is(4321));
         }
     }

core/src/test/java/org/elasticsearch/transport/netty/NettyTransportMultiPortTests.java

@@ -170,7 +170,7 @@ public class NettyTransportMultiPortTests extends ESTestCase {
                         // Set SO_REUSEADDR as we may bind here and not be able
                         // to reuse the address immediately without it.
                         serverSocket.setReuseAddress(NetworkUtils.defaultReuseAddress());
-                        serverSocket.bind(new InetSocketAddress(nextPort));
+                        serverSocket.bind(new InetSocketAddress(InetAddress.getLoopbackAddress(), nextPort));
 
                         // bind was a success
                         logger.debug("port [{}] available.", nextPort);
@@ -199,7 +199,7 @@ public class NettyTransportMultiPortTests extends ESTestCase {
 
     private void assertConnectionRefused(int port) throws Exception {
         try {
-            trySocketConnection(new InetSocketTransportAddress("localhost", port).address());
+            trySocketConnection(new InetSocketTransportAddress(InetAddress.getByName("localhost"), port).address());
             fail("Expected to get exception when connecting to port " + port);
         } catch (IOException e) {
             // expected
@@ -213,7 +213,7 @@ public class NettyTransportMultiPortTests extends ESTestCase {
 
     private void assertPortIsBound(String host, int port) throws Exception {
         logger.info("Trying to connect to [{}]:[{}]", host, port);
-        trySocketConnection(new InetSocketTransportAddress(host, port).address());
+        trySocketConnection(new InetSocketTransportAddress(InetAddress.getByName(host), port).address());
     }
 
     private void trySocketConnection(InetSocketAddress address) throws Exception {

core/src/test/java/org/elasticsearch/transport/netty/NettyTransportTests.java

@@ -0,0 +1,130 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.transport.netty;
+
+import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.test.ESTestCase;
+
+/** Unit tests for NettyTransport */
+public class NettyTransportTests extends ESTestCase {
+    
+    /** Test ipv4 host with a default port works */
+    public void testParseV4DefaultPort() throws Exception {
+        TransportAddress[] addresses = NettyTransport.parse("127.0.0.1", "1234", Integer.MAX_VALUE);
+        assertEquals(1, addresses.length);
+
+        assertEquals("127.0.0.1", addresses[0].getAddress());
+        assertEquals(1234, addresses[0].getPort());
+    }
+
+    /** Test ipv4 host with a default port range works */
+    public void testParseV4DefaultRange() throws Exception {
+        TransportAddress[] addresses = NettyTransport.parse("127.0.0.1", "1234-1235", Integer.MAX_VALUE);
+        assertEquals(2, addresses.length);
+
+        assertEquals("127.0.0.1", addresses[0].getAddress());
+        assertEquals(1234, addresses[0].getPort());
+        
+        assertEquals("127.0.0.1", addresses[1].getAddress());
+        assertEquals(1235, addresses[1].getPort());
+    }
+
+    /** Test ipv4 host with port works */
+    public void testParseV4WithPort() throws Exception {
+        TransportAddress[] addresses = NettyTransport.parse("127.0.0.1:2345", "1234", Integer.MAX_VALUE);
+        assertEquals(1, addresses.length);
+
+        assertEquals("127.0.0.1", addresses[0].getAddress());
+        assertEquals(2345, addresses[0].getPort());
+    }
+
+    /** Test ipv4 host with port range works */
+    public void testParseV4WithPortRange() throws Exception {
+        TransportAddress[] addresses = NettyTransport.parse("127.0.0.1:2345-2346", "1234", Integer.MAX_VALUE);
+        assertEquals(2, addresses.length);
+
+        assertEquals("127.0.0.1", addresses[0].getAddress());
+        assertEquals(2345, addresses[0].getPort());
+
+        assertEquals("127.0.0.1", addresses[1].getAddress());
+        assertEquals(2346, addresses[1].getPort());
+    }
+
+    /** Test unbracketed ipv6 hosts in configuration fail. Leave no ambiguity */
+    public void testParseV6UnBracketed() throws Exception {
+        try {
+            NettyTransport.parse("::1", "1234", Integer.MAX_VALUE);
+            fail("should have gotten exception");
+        } catch (IllegalArgumentException expected) {
+            assertTrue(expected.getMessage().contains("must be bracketed"));
+        }
+    }
+
+    /** Test ipv6 host with a default port works */
+    public void testParseV6DefaultPort() throws Exception {
+        TransportAddress[] addresses = NettyTransport.parse("[::1]", "1234", Integer.MAX_VALUE);
+        assertEquals(1, addresses.length);
+
+        assertEquals("::1", addresses[0].getAddress());
+        assertEquals(1234, addresses[0].getPort());
+    }
+
+    /** Test ipv6 host with a default port range works */
+    public void testParseV6DefaultRange() throws Exception {
+        TransportAddress[] addresses = NettyTransport.parse("[::1]", "1234-1235", Integer.MAX_VALUE);
+        assertEquals(2, addresses.length);
+
+        assertEquals("::1", addresses[0].getAddress());
+        assertEquals(1234, addresses[0].getPort());
+        
+        assertEquals("::1", addresses[1].getAddress());
+        assertEquals(1235, addresses[1].getPort());
+    }
+
+    /** Test ipv6 host with port works */
+    public void testParseV6WithPort() throws Exception {
+        TransportAddress[] addresses = NettyTransport.parse("[::1]:2345", "1234", Integer.MAX_VALUE);
+        assertEquals(1, addresses.length);
+
+        assertEquals("::1", addresses[0].getAddress());
+        assertEquals(2345, addresses[0].getPort());
+    }
+
+    /** Test ipv6 host with port range works */
+    public void testParseV6WithPortRange() throws Exception {
+        TransportAddress[] addresses = NettyTransport.parse("[::1]:2345-2346", "1234", Integer.MAX_VALUE);
+        assertEquals(2, addresses.length);
+
+        assertEquals("::1", addresses[0].getAddress());
+        assertEquals(2345, addresses[0].getPort());
+
+        assertEquals("::1", addresses[1].getAddress());
+        assertEquals(2346, addresses[1].getPort());
+    }
+    
+    /** Test per-address limit */
+    public void testAddressLimit() throws Exception {
+        TransportAddress[] addresses = NettyTransport.parse("[::1]:100-200", "1000", 3);
+        assertEquals(3, addresses.length);
+        assertEquals(100, addresses[0].getPort());
+        assertEquals(101, addresses[1].getPort());
+        assertEquals(102, addresses[2].getPort());
+    }
+}

core/src/test/java/org/elasticsearch/transport/netty/SimpleNettyTransportTests.java

@@ -31,6 +31,9 @@ import org.elasticsearch.transport.AbstractSimpleTransportTests;
 import org.elasticsearch.transport.ConnectTransportException;
 import org.junit.Test;
 
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+
 public class SimpleNettyTransportTests extends AbstractSimpleTransportTests {
 
     @Override
@@ -44,7 +47,7 @@ public class SimpleNettyTransportTests extends AbstractSimpleTransportTests {
     }
 
     @Test(expected = ConnectTransportException.class)
-    public void testConnectException() {
-        serviceA.connectToNode(new DiscoveryNode("C", new InetSocketTransportAddress("localhost", 9876), Version.CURRENT));
+    public void testConnectException() throws UnknownHostException {
+        serviceA.connectToNode(new DiscoveryNode("C", new InetSocketTransportAddress(InetAddress.getByName("localhost"), 9876), Version.CURRENT));
     }
 }

core/src/test/java/org/elasticsearch/tribe/TribeIT.java

@@ -70,7 +70,7 @@ public class TribeIT extends ESIntegTestCase {
     public static void setupSecondCluster() throws Exception {
         ESIntegTestCase.beforeClass();
         // create another cluster
-        cluster2 = new InternalTestCluster(randomLong(), createTempDir(), 2, 2, Strings.randomBase64UUID(getRandom()), 0, false, SECOND_CLUSTER_NODE_PREFIX);
+        cluster2 = new InternalTestCluster(InternalTestCluster.configuredNodeMode(), randomLong(), createTempDir(), 2, 2, Strings.randomBase64UUID(getRandom()), 0, false, SECOND_CLUSTER_NODE_PREFIX);
         cluster2.beforeTest(getRandom(), 0.1);
         cluster2.ensureAtLeastNumDataNodes(2);
     }

core/src/test/java/org/elasticsearch/tribe/TribeUnitTests.java

@@ -48,7 +48,7 @@ public class TribeUnitTests extends ESTestCase {
     private static Node tribe1;
     private static Node tribe2;
 
-    private static final String NODE_MODE = InternalTestCluster.nodeMode();
+    private static final String NODE_MODE = InternalTestCluster.configuredNodeMode();
 
     @BeforeClass
     public static void createTribes() {

dev-tools/build_release.py

@@ -420,7 +420,7 @@ def smoke_test_release(release, files, expected_hash, plugins):
 
     background = '-d'
     print('  Starting elasticsearch deamon from [%s]' % os.path.join(tmp_dir, 'elasticsearch-%s' % release))
-    run('%s; %s -Des.node.name=smoke_tester -Des.cluster.name=prepare_release -Des.discovery.zen.ping.multicast.enabled=false -Des.script.inline=on -Des.script.indexed=on %s'
+    run('%s; %s -Des.node.name=smoke_tester -Des.cluster.name=prepare_release -Des.script.inline=on -Des.script.indexed=on %s'
          % (java_exe(), es_run_path, background))
     conn = HTTPConnection('127.0.0.1', 9200, 20);
     wait_for_node_startup()

dev-tools/create_bwc_index.py

@@ -141,7 +141,6 @@ def start_node(version, release_dir, data_dir, repo_dir, tcp_port=DEFAULT_TRANSP
     '-Des.path.logs=logs',
     '-Des.cluster.name=%s' % cluster_name,
     '-Des.network.host=localhost',
-    '-Des.discovery.zen.ping.multicast.enabled=false',
     '-Des.transport.tcp.port=%s' % tcp_port,
     '-Des.http.port=%s' % http_port,
     '-Des.path.repo=%s' % repo_dir

dev-tools/src/main/resources/ant/integration-tests.xml

@@ -136,8 +136,7 @@
       <attribute name="home" default="${integ.scratch}/elasticsearch-${elasticsearch.version}"/>
       <attribute name="spawn" default="true"/>
       <attribute name="args" default="${integ.args}"/>
-      <attribute name="es.unicast.enabled" default="false"/>
-      <attribute name="es.unicast.hosts" default=""/>
+      <attribute name="es.unicast.hosts" default="localhost:${integ.transport.port}"/>
       <attribute name="es.cluster.name" default="${integ.cluster.name}"/>
       <attribute name="es.http.port" default="${integ.http.port}"/>
       <attribute name="es.transport.tcp.port" default="${integ.transport.port}"/>
@@ -166,11 +165,9 @@
           <arg value="-Des.http.port=@{es.http.port}"/>
           <arg value="-Des.transport.tcp.port=@{es.transport.tcp.port}"/>
           <arg value="-Des.pidfile=@{es.pidfile}"/>
-          <arg value="-Des.discovery.zen.ping.unicast.enabled=@{es.unicast.enabled}"/>
           <arg value="-Des.discovery.zen.ping.unicast.hosts=@{es.unicast.hosts}"/>
           <arg value="-Des.path.repo=@{home}/repo"/>
           <arg value="-Des.path.shared_data=@{home}/../"/>
-          <arg value="-Des.discovery.zen.ping.multicast.enabled=false"/>
           <arg value="-Des.script.inline=on"/>
           <arg value="-Des.script.indexed=on"/>
           <arg value="-Des.repositories.url.allowed_urls=http://snapshot.test*"/>
@@ -243,7 +240,7 @@
     <attribute name="es.pidfile" default="${integ.pidfile}"/>
     <attribute name="es.peer.list" />
     <sequential>
-    <startup-elasticsearch es.pidfile="@{es.pidfile}" es.unicast.enabled="true"
+    <startup-elasticsearch es.pidfile="@{es.pidfile}"
                            es.transport.tcp.port="@{es.transport.port}" es.http.port="@{es.http.port}"
                            es.unicast.hosts="@{es.peer.list}"/>
     </sequential>

dev-tools/src/main/resources/forbidden/all-signatures.txt

@@ -59,3 +59,30 @@ java.nio.file.Files#isHidden(java.nio.file.Path) @ Dependent on the operating sy
 
 java.nio.file.Files#getFileStore(java.nio.file.Path) @ Use Environment.getFileStore() instead, impacted by JDK-8034057
 java.nio.file.Files#isWritable(java.nio.file.Path) @ Use Environment.isWritable() instead, impacted by JDK-8034057
+
+@defaultMessage Resolve hosts explicitly to the address(es) you want with InetAddress.
+java.net.InetSocketAddress#<init>(java.lang.String,int)
+java.net.Socket#<init>(java.lang.String,int)
+java.net.Socket#<init>(java.lang.String,int,java.net.InetAddress,int)
+
+@defaultMessage Don't bind to wildcard addresses. Be specific.
+java.net.DatagramSocket#<init>()
+java.net.DatagramSocket#<init>(int)
+java.net.InetSocketAddress#<init>(int)
+java.net.MulticastSocket#<init>()
+java.net.MulticastSocket#<init>(int)
+java.net.ServerSocket#<init>(int)
+java.net.ServerSocket#<init>(int,int)
+
+@defaultMessage use NetworkAddress format/formatAddress to print IP or IP+ports
+java.net.InetAddress#toString()
+java.net.InetAddress#getHostAddress()
+java.net.Inet4Address#getHostAddress()
+java.net.Inet6Address#getHostAddress()
+java.net.InetSocketAddress#toString()
+
+@defaultMessage avoid DNS lookups by accident: if you have a valid reason, then @SuppressWarnings with that reason so its completely clear
+java.net.InetAddress#getHostName()
+java.net.InetAddress#getCanonicalHostName()
+
+java.net.InetSocketAddress#getHostName() @ Use getHostString() instead, which avoids a DNS lookup

dev-tools/upgrade-tests.py

@@ -105,8 +105,7 @@ def start_node(version, data_dir, node_dir, unicast_host_list, tcp_port, http_po
     foreground = ''
   return subprocess.Popen([es_run_path,
     '-Des.path.data=%s' % data_dir, '-Des.cluster.name=upgrade_test',  
-    '-Des.discovery.zen.ping.unicast.hosts=%s' % unicast_host_list, 
-    '-Des.discovery.zen.ping.multicast.enabled=false',
+    '-Des.discovery.zen.ping.unicast.hosts=%s' % unicast_host_list,
     '-Des.transport.tcp.port=%s' % tcp_port,
     '-Des.http.port=%s' % http_port,
     foreground], stdout=subprocess.PIPE, stderr=subprocess.PIPE)

distribution/src/main/resources/config/elasticsearch.yml

@@ -71,13 +71,10 @@
 #
 # --------------------------------- Discovery ----------------------------------
 #
-# Elasticsearch nodes will find each other via multicast, by default.
-#
-# To use the unicast discovery, disable the multicast discovery:
-#
-# discovery.zen.ping.multicast.enabled: false
+# Elasticsearch nodes will find each other via unicast, by default.
 #
 # Pass an initial list of hosts to perform discovery when new node is started:
+# The default list of hosts is ["127.0.0.1", "[::1]"]
 #
 # discovery.zen.ping.unicast.hosts: ["host1", "host2"]
 #

docs/plugins/delete-by-query.asciidoc

@@ -6,7 +6,7 @@ The delete-by-query plugin adds support for deleting all of the documents
 replacement for the problematic _delete-by-query_ functionality which has been
 removed from Elasticsearch core.
 
-Internally, it uses the {ref}/search-request-scroll.html#scroll-scan[Scan/Scroll]
+Internally, it uses {ref}/search-request-scroll.html[Scroll]
 and {ref}/docs-bulk.html[Bulk] APIs to delete documents in an efficient and
 safe manner. It is slower than the old _delete-by-query_ functionality, but
 fixes the problems with the previous implementation.
@@ -101,7 +101,7 @@ See {ref}/search-uri-request.html[URI search request] for details.
 
 `size`::
 
-The number of hits returned *per shard* by the {ref}/search-request-scroll.html#scroll-scan[scan]
+The number of hits returned by the {ref}/search-request-scroll.html[scroll]
 request.  Defaults to 10.  May also be specified in the request body.
 
 `timeout`::
@@ -148,7 +148,7 @@ The JSON response looks like this:
 --------------------------------------------------
 
 Internally, the query is used to execute an initial
-{ref}/search-request-scroll.html#scroll-scan[scroll/scan] request. As hits are
+{ref}/search-request-scroll.html[scroll] request. As hits are
 pulled from the scroll API, they are passed to the {ref}/docs-bulk.html[Bulk
 API] for deletion.
 
@@ -157,7 +157,7 @@ was visible to search at the time the request was executed.  Any documents
 that have been reindexed or updated during execution will not be deleted.
 
 Since documents can be updated or deleted by external operations during the
-_scan-scroll-bulk_ process, the plugin keeps track of different counters for
+_scroll-bulk_ process, the plugin keeps track of different counters for
 each index, with the totals displayed under the `_all` index.  The counters
 are as follows:
 
@@ -212,7 +212,7 @@ Resiliency::
 === New delete-by-query implementation
 
 The new implementation, provided by this plugin, is built internally
-using  {ref}/search-request-scroll.html#scroll-scan[scan and scroll] to return
+using  {ref}/search-request-scroll.html[scroll] to return
 the document IDs and versions of all the documents that need to be deleted.
 It then uses  the {ref}/docs-bulk.html[`bulk` API] to do the actual deletion.
 
@@ -231,8 +231,8 @@ try-once::
 
 syntactic sugar::
 
-    A delete-by-query is equivalent to a scan/scroll search and corresponding
-    bulk-deletes by ID.
+    A delete-by-query is equivalent to a scroll search ordered by `_doc` and
+    corresponding bulk-deletes by ID.
 
 point-in-time::
 
@@ -267,4 +267,4 @@ move the functionality to a plugin instead of replacing the feautre in core:
 * There is currently no way to monitor or cancel a running delete-by-query
   request, except for the `timeout` parameter.
 
-We have plans to solve both of these issues in a later version of Elasticsearch.
+We have plans to solve both of these issues in a later version of Elasticsearch.

docs/plugins/discovery-multicast.asciidoc

@@ -0,0 +1,55 @@
+[[discovery-multicast]]
+=== Multicast Discovery Plugin
+
+The Multicast Discovery plugin provides the ability to form a cluster using
+TCP/IP multicast messages.
+
+[[discovery-multicast-install]]
+[float]
+==== Installation
+
+This plugin can be installed using the plugin manager:
+
+[source,sh]
+----------------------------------------------------------------
+sudo bin/plugin install discovery-multicast
+----------------------------------------------------------------
+
+The plugin must be installed on every node in the cluster, and each node must
+be restarted after installation.
+
+[[discovery-multicast-remove]]
+[float]
+==== Removal
+
+The plugin can be removed with the following command:
+
+[source,sh]
+----------------------------------------------------------------
+sudo bin/plugin remove discovery-multicast
+----------------------------------------------------------------
+
+The node must be stopped before removing the plugin.
+
+[[discovery-multicast-usage]]
+==== Configuring multicast discovery
+
+Multicast ping discovery of other nodes is done by sending one or more
+multicast requests which existing nodes will receive and
+respond to. It provides the following settings with the
+`discovery.zen.ping.multicast` prefix:
+
+[cols="<,<",options="header",]
+|=======================================================================
+|Setting |Description
+|`group` |The group address to use. Defaults to `224.2.2.4`.
+
+|`port` |The port to use. Defaults to `54328`.
+
+|`ttl` |The ttl of the multicast message. Defaults to `3`.
+
+|`address` |The address to bind to, defaults to `null` which means it
+will bind `network.bind_host`
+
+|`enabled` |Whether multicast ping discovery is enabled. Defaults to `false`.
+|=======================================================================

docs/plugins/discovery.asciidoc

@@ -26,6 +26,10 @@ support for using Azure as a repository for
 
 The Google Compute Engine Cloud plugin uses the GCE API for unicast discovery.
 
+<<discovery-multicast,Multicast>>::
+
+The multicast plugin sends multicast messages to discover other nodes in the cluster.
+
 [float]
 ==== Community contributed discovery plugins
 
@@ -41,5 +45,7 @@ include::cloud-azure.asciidoc[]
 
 include::cloud-gce.asciidoc[]
 
+include::discovery-multicast.asciidoc[]
+