Merge branch 'master' into feature/query-refactoring

2015-07-31 10:56:05 +02:00 · 2015-07-31 10:56:05 +02:00 · 7f28dc14ee
parent af75117845 8efa18e616
commit 7f28dc14ee
35 changed files with 1100 additions and 291 deletions
--- a/.gitignore
+++ b/.gitignore
@ -21,13 +21,8 @@ backwards/
 ## files to ensure common coding style across Eclipse and IDEA.
 .project
 .classpath
-/.settings
 eclipse-build
-*/.project
-*/.classpath
-*/eclipse-build
-*/.settings
-plugins/*/.settings
+.settings

 ## netbeans ignores
 nb-configuration.xml
--- a/core/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java
+++ b/core/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java
@ -19,12 +19,13 @@

 package org.elasticsearch.bootstrap;

-import org.apache.lucene.util.StringHelper;
 import org.apache.lucene.util.Constants;
+import org.apache.lucene.util.StringHelper;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.Version;
 import org.elasticsearch.common.PidFile;
 import org.elasticsearch.common.SuppressForbidden;
+import org.elasticsearch.common.cli.CliTool;
 import org.elasticsearch.common.cli.Terminal;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.inject.CreationException;
@ -222,10 +223,17 @@ public class Bootstrap {
    }

    public static void main(String[] args) {
+        BootstrapCLIParser bootstrapCLIParser = new BootstrapCLIParser();
+        CliTool.ExitStatus status = bootstrapCLIParser.execute(args);
+
+        if (CliTool.ExitStatus.OK != status) {
+            System.exit(status.status());
+        }
+
        System.setProperty("es.logger.prefix", "");
        INSTANCE = new Bootstrap();

-        boolean foreground = System.getProperty("es.foreground", System.getProperty("es-foreground")) != null;
+        boolean foreground = !"false".equals(System.getProperty("es.foreground", System.getProperty("es-foreground")));
        // handle the wrapper system property, if its a service, don't run as a service
        if (System.getProperty("wrapper.service", "XXX").equalsIgnoreCase("true")) {
            foreground = false;
--- a/core/src/main/java/org/elasticsearch/bootstrap/BootstrapCLIParser.java
+++ b/core/src/main/java/org/elasticsearch/bootstrap/BootstrapCLIParser.java
@ -0,0 +1,167 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.bootstrap;
+
+import com.google.common.base.Strings;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
+import org.elasticsearch.Build;
+import org.elasticsearch.common.cli.CliTool;
+import org.elasticsearch.common.cli.CliToolConfig;
+import org.elasticsearch.common.cli.Terminal;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.monitor.jvm.JvmInfo;
+
+import java.util.Iterator;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Properties;
+
+import static org.elasticsearch.common.cli.CliToolConfig.Builder.cmd;
+import static org.elasticsearch.common.cli.CliToolConfig.Builder.optionBuilder;
+
+public class BootstrapCLIParser extends CliTool {
+
+    private static final CliToolConfig CONFIG = CliToolConfig.config("elasticsearch", BootstrapCLIParser.class)
+            .cmds(Start.CMD, Version.CMD)
+            .build();
+
+    public BootstrapCLIParser() {
+        super(CONFIG);
+    }
+
+    public BootstrapCLIParser(Terminal terminal) {
+        super(CONFIG, terminal);
+    }
+
+    @Override
+    protected Command parse(String cmdName, CommandLine cli) throws Exception {
+        switch (cmdName.toLowerCase(Locale.ROOT)) {
+            case Start.NAME:
+                return Start.parse(terminal, cli);
+            case Version.NAME:
+                return Version.parse(terminal, cli);
+            default:
+                assert false : "should never get here, if the user enters an unknown command, an error message should be shown before parse is called";
+                return null;
+        }
+    }
+
+    static class Version extends CliTool.Command {
+
+        private static final String NAME = "version";
+
+        private static final CliToolConfig.Cmd CMD = cmd(NAME, Version.class).build();
+
+        public static Command parse(Terminal terminal, CommandLine cli) {
+            return new Version(terminal);
+        }
+
+        public Version(Terminal terminal) {
+            super(terminal);
+        }
+
+        @Override
+        public ExitStatus execute(Settings settings, Environment env) throws Exception {
+            terminal.println("Version: %s, Build: %s/%s, JVM: %s", org.elasticsearch.Version.CURRENT, Build.CURRENT.hashShort(), Build.CURRENT.timestamp(), JvmInfo.jvmInfo().version());
+            return ExitStatus.OK_AND_EXIT;
+        }
+    }
+
+    static class Start extends CliTool.Command {
+
+        private static final String NAME = "start";
+
+        private static final CliToolConfig.Cmd CMD = cmd(NAME, Start.class)
+                .options(
+                        optionBuilder("d", "daemonize").hasArg(false).required(false),
+                        optionBuilder("p", "pidfile").hasArg(true).required(false),
+                        optionBuilder("V", "version").hasArg(false).required(false),
+                        Option.builder("D").argName("property=value").valueSeparator('=').numberOfArgs(2)
+                )
+                .stopAtNonOption(true) // needed to parse the --foo.bar options, so this parser must be lenient
+                .build();
+
+        public static Command parse(Terminal terminal, CommandLine cli) {
+            if (cli.hasOption("V")) {
+                return Version.parse(terminal, cli);
+            }
+
+            if (cli.hasOption("d")) {
+                System.setProperty("es.foreground", "false");
+            }
+
+            String pidFile = cli.getOptionValue("pidfile");
+            if (!Strings.isNullOrEmpty(pidFile)) {
+                System.setProperty("es.pidfile", pidFile);
+            }
+
+            if (cli.hasOption("D")) {
+                Properties properties = cli.getOptionProperties("D");
+                for (Map.Entry<Object, Object> entry : properties.entrySet()) {
+                    String key = (String) entry.getKey();
+                    String propertyName = key.startsWith("es.") ? key : "es." + key;
+                    System.setProperty(propertyName, entry.getValue().toString());
+                }
+            }
+
+            // hacky way to extract all the fancy extra args, there is no CLI tool helper for this
+            Iterator<String> iterator = cli.getArgList().iterator();
+            while (iterator.hasNext()) {
+                String arg = iterator.next();
+                if (!arg.startsWith("--")) {
+                    throw new IllegalArgumentException("Parameter [" + arg + "]does not start with --");
+                }
+                // if there is no = sign, we have to get the next argu
+                arg = arg.replace("--", "");
+                if (arg.contains("=")) {
+                    String[] splitArg = arg.split("=", 2);
+                    String key = splitArg[0];
+                    String value = splitArg[1];
+                    System.setProperty("es." + key, value);
+                } else {
+                    if (iterator.hasNext()) {
+                        String value = iterator.next();
+                        if (value.startsWith("--")) {
+                            throw new IllegalArgumentException("Parameter [" + arg + "] needs value");
+                        }
+                        System.setProperty("es." + arg, value);
+                    } else {
+                        throw new IllegalArgumentException("Parameter [" + arg + "] needs value");
+                    }
+                }
+            }
+
+            return new Start(terminal);
+        }
+
+        public Start(Terminal terminal) {
+            super(terminal);
+
+        }
+
+        @Override
+        public ExitStatus execute(Settings settings, Environment env) throws Exception {
+            return ExitStatus.OK;
+        }
+    }
+
+}
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java
@ -51,7 +51,7 @@ import static com.google.common.collect.Maps.newHashMap;
 public class IndexNameExpressionResolver extends AbstractComponent {

    private final ImmutableList<ExpressionResolver> expressionResolvers;
-    private DateMathExpressionResolver dateMathExpressionResolver;
+    private final DateMathExpressionResolver dateMathExpressionResolver;

    @Inject
    public IndexNameExpressionResolver(Settings settings) {
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDecider.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDecider.java
@ -19,6 +19,7 @@

 package org.elasticsearch.cluster.routing.allocation.decider;

+import com.google.common.collect.Sets;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.ClusterInfo;
@ -37,6 +38,7 @@ import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.node.settings.NodeSettingsService;

 import java.util.Map;
+import java.util.Set;

 /**
 * The {@link DiskThresholdDecider} checks that the node a shard is potentially
@ -128,6 +130,8 @@ public class DiskThresholdDecider extends AllocationDecider {
     */
    class DiskListener implements ClusterInfoService.Listener {
        private final Client client;
+        private final Set<String> nodeHasPassedWatermark = Sets.newConcurrentHashSet();
+
        private long lastRunNS;

        DiskListener(Client client) {
@ -162,21 +166,55 @@ public class DiskThresholdDecider extends AllocationDecider {
            Map<String, DiskUsage> usages = info.getNodeDiskUsages();
            if (usages != null) {
                boolean reroute = false;
-                for (DiskUsage entry : usages.values()) {
-                    warnAboutDiskIfNeeded(entry);
-                    if (entry.getFreeBytes() < DiskThresholdDecider.this.freeBytesThresholdHigh.bytes() ||
-                            entry.getFreeDiskAsPercentage() < DiskThresholdDecider.this.freeDiskThresholdHigh) {
+                String explanation = "";
+
+                // Garbage collect nodes that have been removed from the cluster
+                // from the map that tracks watermark crossing
+                Set<String> nodes = usages.keySet();
+                for (String node : nodeHasPassedWatermark) {
+                    if (nodes.contains(node) == false) {
+                        nodeHasPassedWatermark.remove(node);
+                    }
+                }
+
+                for (Map.Entry<String, DiskUsage> entry : usages.entrySet()) {
+                    String node = entry.getKey();
+                    DiskUsage usage = entry.getValue();
+                    warnAboutDiskIfNeeded(usage);
+                    if (usage.getFreeBytes() < DiskThresholdDecider.this.freeBytesThresholdHigh.bytes() ||
+                            usage.getFreeDiskAsPercentage() < DiskThresholdDecider.this.freeDiskThresholdHigh) {
                        if ((System.nanoTime() - lastRunNS) > DiskThresholdDecider.this.rerouteInterval.nanos()) {
                            lastRunNS = System.nanoTime();
                            reroute = true;
+                            explanation = "high disk watermark exceeded on one or more nodes";
                        } else {
                            logger.debug("high disk watermark exceeded on {} but an automatic reroute has occurred in the last [{}], skipping reroute",
-                                    entry, DiskThresholdDecider.this.rerouteInterval);
+                                    node, DiskThresholdDecider.this.rerouteInterval);
+                        }
+                        nodeHasPassedWatermark.add(node);
+                    } else if (usage.getFreeBytes() < DiskThresholdDecider.this.freeBytesThresholdLow.bytes() ||
+                            usage.getFreeDiskAsPercentage() < DiskThresholdDecider.this.freeDiskThresholdLow) {
+                        nodeHasPassedWatermark.add(node);
+                    } else {
+                        if (nodeHasPassedWatermark.contains(node)) {
+                            // The node has previously been over the high or
+                            // low watermark, but is no longer, so we should
+                            // reroute so any unassigned shards can be allocated
+                            // if they are able to be
+                            if ((System.nanoTime() - lastRunNS) > DiskThresholdDecider.this.rerouteInterval.nanos()) {
+                                lastRunNS = System.nanoTime();
+                                reroute = true;
+                                explanation = "one or more nodes has gone under the high or low watermark";
+                                nodeHasPassedWatermark.remove(node);
+                            } else {
+                                logger.debug("{} has gone below a disk threshold, but an automatic reroute has occurred in the last [{}], skipping reroute",
+                                        node, DiskThresholdDecider.this.rerouteInterval);
+                            }
                        }
                    }
                }
                if (reroute) {
-                    logger.info("high disk watermark exceeded on one or more nodes, rerouting shards");
+                    logger.info("rerouting shards: [{}]", explanation);
                    // Execute an empty reroute, but don't block on the response
                    client.admin().cluster().prepareReroute().execute();
                }
--- a/core/src/main/java/org/elasticsearch/common/cli/CliTool.java
+++ b/core/src/main/java/org/elasticsearch/common/cli/CliTool.java
@ -22,6 +22,7 @@ package org.elasticsearch.common.cli;
 import com.google.common.base.Preconditions;
 import org.apache.commons.cli.CommandLine;
 import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.DefaultParser;
 import org.apache.commons.cli.GnuParser;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.settings.Settings;
@ -54,6 +55,7 @@ public abstract class CliTool {
    // based on sysexits.h
    public static enum ExitStatus {
        OK(0),
+        OK_AND_EXIT(0),
        USAGE(64),          /* command line usage error */
        DATA_ERROR(65),     /* data format error */
        NO_INPUT(66),       /* cannot open input */
@ -77,6 +79,16 @@ public abstract class CliTool {
        public int status() {
            return status;
        }
+
+        public static ExitStatus fromStatus(int status) {
+            for (ExitStatus exitStatus : values()) {
+                if (exitStatus.status() == status) {
+                    return exitStatus;
+                }
+            }
+
+            return null;
+        }
    }

    protected final Terminal terminal;
@ -98,14 +110,14 @@ public abstract class CliTool {
        env = tuple.v2();
    }

-    public final int execute(String... args) {
+    public final ExitStatus execute(String... args) {

        // first lets see if the user requests tool help. We're doing it only if
        // this is a multi-command tool. If it's a single command tool, the -h/--help
        // option will be taken care of on the command level
        if (!config.isSingle() && args.length > 0 && (args[0].equals("-h") || args[0].equals("--help"))) {
            config.printUsage(terminal);
-            return ExitStatus.OK.status;
+            return ExitStatus.OK_AND_EXIT;
        }

        CliToolConfig.Cmd cmd;
@ -116,14 +128,14 @@ public abstract class CliTool {
            if (args.length == 0) {
                terminal.printError("command not specified");
                config.printUsage(terminal);
-                return ExitStatus.USAGE.status;
+                return ExitStatus.USAGE;
            }

            String cmdName = args[0];
            cmd = config.cmd(cmdName);
            if (cmd == null) {
                terminal.printError("unknown command [%s]. Use [-h] option to list available commands", cmdName);
-                return ExitStatus.USAGE.status;
+                return ExitStatus.USAGE;
            }

            // we now remove the command name from the args
@ -140,20 +152,19 @@ public abstract class CliTool {
        try {

            command = parse(cmd, args);
-            return command.execute(settings, env).status;
-
+            return command.execute(settings, env);
        } catch (IOException ioe) {
            terminal.printError(ioe);
-            return ExitStatus.IO_ERROR.status;
+            return ExitStatus.IO_ERROR;
        } catch (IllegalArgumentException ilae) {
            terminal.printError(ilae);
-            return ExitStatus.USAGE.status;
+            return ExitStatus.USAGE;
        } catch (Throwable t) {
            terminal.printError(t);
            if (command == null) {
-                return ExitStatus.USAGE.status;
+                return ExitStatus.USAGE;
            }
-            return ExitStatus.CODE_ERROR.status;
+            return ExitStatus.CODE_ERROR;
        }
    }

@ -163,12 +174,12 @@ public abstract class CliTool {
    }

    public Command parse(CliToolConfig.Cmd cmd, String[] args) throws Exception {
-        CommandLineParser parser = new GnuParser();
+        CommandLineParser parser = new DefaultParser();
        CommandLine cli = parser.parse(CliToolConfig.OptionsSource.HELP.options(), args, true);
        if (cli.hasOption("h")) {
            return helpCmd(cmd);
        }
-        cli = parser.parse(cmd.options(), args);
+        cli = parser.parse(cmd.options(), args, cmd.isStopAtNonOption());
        Terminal.Verbosity verbosity = Terminal.Verbosity.resolve(cli);
        terminal.verbosity(verbosity);
        return parse(cmd.name(), cli);
@ -210,7 +221,7 @@ public abstract class CliTool {
            @Override
            public ExitStatus execute(Settings settings, Environment env) throws Exception {
                cmd.printUsage(terminal);
-                return ExitStatus.OK;
+                return ExitStatus.OK_AND_EXIT;
            }
        }

--- a/core/src/main/java/org/elasticsearch/common/cli/CliToolConfig.java
+++ b/core/src/main/java/org/elasticsearch/common/cli/CliToolConfig.java
@ -20,6 +20,7 @@
 package org.elasticsearch.common.cli;

 import com.google.common.collect.ImmutableMap;
+import org.apache.commons.cli.CommandLine;
 import org.apache.commons.cli.Option;
 import org.apache.commons.cli.OptionGroup;
 import org.apache.commons.cli.Options;
@ -90,6 +91,10 @@ public class CliToolConfig {
            return new OptionBuilder(shortName, longName);
        }

+        public static Option.Builder optionBuilder(String shortName, String longName) {
+            return Option.builder(shortName).argName(longName).longOpt(longName);
+        }
+
        public static OptionGroupBuilder optionGroup(boolean required) {
            return new OptionGroupBuilder(required);
        }
@ -131,11 +136,13 @@ public class CliToolConfig {
        private final String name;
        private final Class<? extends CliTool.Command> cmdType;
        private final Options options;
+        private final boolean stopAtNonOption;

-        private Cmd(String name, Class<? extends CliTool.Command> cmdType, Options options) {
+        private Cmd(String name, Class<? extends CliTool.Command> cmdType, Options options, boolean stopAtNonOption) {
            this.name = name;
            this.cmdType = cmdType;
            this.options = options;
+            this.stopAtNonOption = stopAtNonOption;
            OptionsSource.VERBOSITY.populate(options);
        }

@ -148,16 +155,11 @@ public class CliToolConfig {
        }

        public Options options() {
-            // TODO Remove this when commons-cli 1.3 will be released
-            // and replace by return options;
-            // See https://issues.apache.org/jira/browse/CLI-183
-            Options copy = new Options();
-            for (Object oOption : options.getOptions()) {
-                Option option = (Option) oOption;
-                copy.addOption(option);
+            return options;
        }
-            OptionsSource.VERBOSITY.populate(copy);
-            return copy;
+
+        public boolean isStopAtNonOption() {
+            return stopAtNonOption;
        }

        public void printUsage(Terminal terminal) {
@ -169,6 +171,7 @@ public class CliToolConfig {
            private final String name;
            private final Class<? extends CliTool.Command> cmdType;
            private Options options = new Options();
+            private boolean stopAtNonOption = false;

            private Builder(String name, Class<? extends CliTool.Command> cmdType) {
                this.name = name;
@ -182,6 +185,13 @@ public class CliToolConfig {
                return this;
            }

+            public Builder options(Option.Builder... optionBuilders) {
+                for (int i = 0; i < optionBuilders.length; i++) {
+                    options.addOption(optionBuilders[i].build());
+                }
+                return this;
+            }
+
            public Builder optionGroups(OptionGroupBuilder... optionGroupBuilders) {
                for (OptionGroupBuilder builder : optionGroupBuilders) {
                    options.addOptionGroup(builder.build());
@ -189,8 +199,19 @@ public class CliToolConfig {
                return this;
            }

+            /**
+              * @param stopAtNonOption if <tt>true</tt> an unrecognized argument stops
+              *     the parsing and the remaining arguments are added to the
+              *     args list. If <tt>false</tt> an unrecognized
+              *     argument triggers a ParseException.
+              */
+            public Builder stopAtNonOption(boolean stopAtNonOption) {
+                this.stopAtNonOption = stopAtNonOption;
+                return this;
+            }
+
            public Cmd build() {
-                return new Cmd(name, cmdType, options);
+                return new Cmd(name, cmdType, options, stopAtNonOption);
            }
        }
    }
--- a/core/src/main/java/org/elasticsearch/plugins/PluginManagerCliParser.java
+++ b/core/src/main/java/org/elasticsearch/plugins/PluginManagerCliParser.java
@ -51,7 +51,7 @@ public class PluginManagerCliParser extends CliTool {
    public static void main(String[] args) {
        Tuple<Settings, Environment> initialSettings = InternalSettingsPreparer.prepareSettings(EMPTY, true, Terminal.DEFAULT);
        LogConfigurator.configure(initialSettings.v1());
-        int status = new PluginManagerCliParser().execute(args);
+        int status = new PluginManagerCliParser().execute(args).status();
        System.exit(status);
    }

--- a/core/src/main/java/org/elasticsearch/plugins/PluginsService.java
+++ b/core/src/main/java/org/elasticsearch/plugins/PluginsService.java
@ -360,7 +360,7 @@ public class PluginsService extends AbstractComponent {
                jars.addAll(bundle.urls);
                JarHell.checkJarHell(jars.toArray(new URL[0]));
            } catch (Exception e) {
-                logger.warn("failed to load bundle {} due to jar hell", bundle.urls);
+                logger.warn("failed to load bundle {} due to jar hell", bundle.urls, e);
            }
            
            // create a child to load the plugins in this bundle
--- a/core/src/main/resources/org/elasticsearch/bootstrap/elasticsearch-start.help
+++ b/core/src/main/resources/org/elasticsearch/bootstrap/elasticsearch-start.help
@ -0,0 +1,26 @@
+NAME
+
+    start - start Elasticsearcion
+
+SYNOPSIS
+
+    elasticsearch start
+
+DESCRIPTION
+
+    This command starts Elasticsearch. You can configure it to run in the foreground, write a pid file
+    and configure arbitrary options that override file-based configuration.
+
+OPTIONS
+
+    -h,--help                    Shows this message
+
+    -p,--pidfile <pidfile>       Creates a pid file in the specified path on start
+
+    -d,--daemonize               Starts Elasticsearch in the background
+
+    -Dproperty=value             Configures an Elasticsearch specific property, like -Dnetwork.host=127.0.0.1
+
+    --property=value             Configures an elasticsearch specific property, like --network.host 127.0.0.1
+    --property value
+
--- a/core/src/main/resources/org/elasticsearch/bootstrap/elasticsearch-version.help
+++ b/core/src/main/resources/org/elasticsearch/bootstrap/elasticsearch-version.help
@ -0,0 +1,16 @@
+NAME
+
+    version - Show version information and exit
+
+SYNOPSIS
+
+    elasticsearch version
+
+DESCRIPTION
+
+    This command shows Elasticsearch version, timestamp and build information as well as JVM info
+
+OPTIONS
+
+    -h,--help                    Shows this message
+
--- a/core/src/main/resources/org/elasticsearch/bootstrap/elasticsearch.help
+++ b/core/src/main/resources/org/elasticsearch/bootstrap/elasticsearch.help
@ -0,0 +1,22 @@
+NAME
+
+    elasticsearch - Manages elasticsearch
+
+SYNOPSIS
+
+    elasticsearch <command>
+
+DESCRIPTION
+
+    Start elasticsearch and manage plugins
+
+COMMANDS
+
+    start      Start elasticsearch
+
+    version    Show version information and exit
+
+NOTES
+
+    [*] For usage help on specific commands please type "elasticsearch <command> -h"
+
--- a/core/src/test/java/org/elasticsearch/bootstrap/BootstrapCliParserTests.java
+++ b/core/src/test/java/org/elasticsearch/bootstrap/BootstrapCliParserTests.java
@ -0,0 +1,239 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.bootstrap;
+
+import org.elasticsearch.Build;
+import org.elasticsearch.Version;
+import org.elasticsearch.common.cli.CliTool.ExitStatus;
+import org.elasticsearch.common.cli.CliToolTestCase;
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.monitor.jvm.JvmInfo;
+import org.hamcrest.Matcher;
+import org.junit.After;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Locale;
+
+import static org.elasticsearch.common.cli.CliTool.ExitStatus.*;
+import static org.hamcrest.Matchers.*;
+
+public class BootstrapCliParserTests extends CliToolTestCase {
+
+    private CaptureOutputTerminal terminal = new CaptureOutputTerminal();
+    private List<String> propertiesToClear = new ArrayList<>();
+
+    @After
+    public void clearProperties() {
+        for (String property : propertiesToClear) {
+            System.clearProperty(property);
+        }
+    }
+
+    public void testThatVersionIsReturned() throws Exception {
+        BootstrapCLIParser parser = new BootstrapCLIParser(terminal);
+        ExitStatus status = parser.execute(args("version"));
+        assertStatus(status, OK_AND_EXIT);
+
+        assertThatTerminalOutput(containsString(Version.CURRENT.toString()));
+        assertThatTerminalOutput(containsString(Build.CURRENT.hashShort()));
+        assertThatTerminalOutput(containsString(Build.CURRENT.timestamp()));
+        assertThatTerminalOutput(containsString(JvmInfo.jvmInfo().version()));
+    }
+
+    public void testThatVersionIsReturnedAsStartParameter() throws Exception {
+        BootstrapCLIParser parser = new BootstrapCLIParser(terminal);
+        ExitStatus status = parser.execute(args("start -V"));
+        assertStatus(status, OK_AND_EXIT);
+
+        assertThatTerminalOutput(containsString(Version.CURRENT.toString()));
+        assertThatTerminalOutput(containsString(Build.CURRENT.hashShort()));
+        assertThatTerminalOutput(containsString(Build.CURRENT.timestamp()));
+        assertThatTerminalOutput(containsString(JvmInfo.jvmInfo().version()));
+
+        CaptureOutputTerminal terminal = new CaptureOutputTerminal();
+        parser = new BootstrapCLIParser(terminal);
+        status = parser.execute(args("start --version"));
+        assertStatus(status, OK_AND_EXIT);
+
+        assertThatTerminalOutput(containsString(Version.CURRENT.toString()));
+        assertThatTerminalOutput(containsString(Build.CURRENT.hashShort()));
+        assertThatTerminalOutput(containsString(Build.CURRENT.timestamp()));
+        assertThatTerminalOutput(containsString(JvmInfo.jvmInfo().version()));
+    }
+
+    public void testThatPidFileCanBeConfigured() throws Exception {
+        BootstrapCLIParser parser = new BootstrapCLIParser(terminal);
+        registerProperties("es.pidfile");
+
+        ExitStatus status = parser.execute(args("start --pidfile")); // missing pid file
+        assertStatus(status, USAGE);
+
+        // good cases
+        status = parser.execute(args("start --pidfile /tmp/pid"));
+        assertStatus(status, OK);
+        assertSystemProperty("es.pidfile", "/tmp/pid");
+
+        System.clearProperty("es.pidfile");
+        status = parser.execute(args("start -p /tmp/pid"));
+        assertStatus(status, OK);
+        assertSystemProperty("es.pidfile", "/tmp/pid");
+    }
+
+    public void testThatParsingDaemonizeWorks() throws Exception {
+        BootstrapCLIParser parser = new BootstrapCLIParser(terminal);
+        registerProperties("es.foreground");
+
+        ExitStatus status = parser.execute(args("start -d"));
+        assertStatus(status, OK);
+        assertThat(System.getProperty("es.foreground"), is("false"));
+    }
+
+    public void testThatNotDaemonizingDoesNotConfigureProperties() throws Exception {
+        BootstrapCLIParser parser = new BootstrapCLIParser(terminal);
+        registerProperties("es.foreground");
+
+        ExitStatus status = parser.execute(args("start"));
+        assertStatus(status, OK);
+        assertThat(System.getProperty("es.foreground"), is(nullValue()));
+    }
+
+    public void testThatJavaPropertyStyleArgumentsCanBeParsed() throws Exception {
+        BootstrapCLIParser parser = new BootstrapCLIParser(terminal);
+        registerProperties("es.foo", "es.spam");
+
+        ExitStatus status = parser.execute(args("start -Dfoo=bar -Dspam=eggs"));
+        assertStatus(status, OK);
+        assertSystemProperty("es.foo", "bar");
+        assertSystemProperty("es.spam", "eggs");
+    }
+
+    public void testThatJavaPropertyStyleArgumentsWithEsPrefixAreNotPrefixedTwice() throws Exception {
+        BootstrapCLIParser parser = new BootstrapCLIParser(terminal);
+        registerProperties("es.spam", "es.pidfile");
+
+        ExitStatus status = parser.execute(args("start -Des.pidfile=/path/to/foo/elasticsearch/distribution/zip/target/integ-tests/es.pid -Dspam=eggs"));
+        assertStatus(status, OK);
+        assertThat(System.getProperty("es.es.pidfile"), is(nullValue()));
+        assertSystemProperty("es.pidfile", "/path/to/foo/elasticsearch/distribution/zip/target/integ-tests/es.pid");
+        assertSystemProperty("es.spam", "eggs");
+    }
+
+    public void testThatUnknownLongOptionsCanBeParsed() throws Exception {
+        BootstrapCLIParser parser = new BootstrapCLIParser(terminal);
+        registerProperties("es.network.host", "es.my.option");
+
+        ExitStatus status = parser.execute(args("start --network.host 127.0.0.1 --my.option=true"));
+        assertStatus(status, OK);
+        assertSystemProperty("es.network.host", "127.0.0.1");
+        assertSystemProperty("es.my.option", "true");
+    }
+
+    public void testThatUnknownLongOptionsNeedAValue() throws Exception {
+        BootstrapCLIParser parser = new BootstrapCLIParser(terminal);
+        registerProperties("es.network.host");
+
+        ExitStatus status = parser.execute(args("start --network.host"));
+        assertStatus(status, USAGE);
+        assertThatTerminalOutput(containsString("Parameter [network.host] needs value"));
+
+        status = parser.execute(args("start --network.host --foo"));
+        assertStatus(status, USAGE);
+        assertThatTerminalOutput(containsString("Parameter [network.host] needs value"));
+    }
+
+    public void testParsingErrors() {
+        BootstrapCLIParser parser = new BootstrapCLIParser(terminal);
+
+        // unknown params
+        ExitStatus status = parser.execute(args("version --unknown-param /tmp/pid"));
+        assertStatus(status, USAGE);
+        assertThatTerminalOutput(containsString("Unrecognized option: --unknown-param"));
+
+        // single dash in extra params
+        terminal = new CaptureOutputTerminal();
+        parser = new BootstrapCLIParser(terminal);
+        status = parser.execute(args("start -network.host 127.0.0.1"));
+        assertStatus(status, USAGE);
+        assertThatTerminalOutput(containsString("Parameter [-network.host]does not start with --"));
+
+        // never ended parameter
+        terminal = new CaptureOutputTerminal();
+        parser = new BootstrapCLIParser(terminal);
+        status = parser.execute(args("start --network.host"));
+        assertStatus(status, USAGE);
+        assertThatTerminalOutput(containsString("Parameter [network.host] needs value"));
+
+        // free floating value
+        terminal = new CaptureOutputTerminal();
+        parser = new BootstrapCLIParser(terminal);
+        status = parser.execute(args("start 127.0.0.1"));
+        assertStatus(status, USAGE);
+        assertThatTerminalOutput(containsString("Parameter [127.0.0.1]does not start with --"));
+    }
+
+    public void testHelpWorks() throws Exception {
+        List<Tuple<String, String>> tuples = new ArrayList<>();
+        tuples.add(new Tuple<>("version --help", "elasticsearch-version.help"));
+        tuples.add(new Tuple<>("version -h", "elasticsearch-version.help"));
+        tuples.add(new Tuple<>("start --help", "elasticsearch-start.help"));
+        tuples.add(new Tuple<>("start -h", "elasticsearch-start.help"));
+        tuples.add(new Tuple<>("--help", "elasticsearch.help"));
+        tuples.add(new Tuple<>("-h", "elasticsearch.help"));
+
+        for (Tuple<String, String> tuple : tuples) {
+            terminal = new CaptureOutputTerminal();
+            BootstrapCLIParser parser = new BootstrapCLIParser(terminal);
+            ExitStatus status = parser.execute(args(tuple.v1()));
+            assertStatus(status, OK_AND_EXIT);
+            assertTerminalOutputContainsHelpFile(terminal, "/org/elasticsearch/bootstrap/" + tuple.v2());
+        }
+    }
+
+    public void testThatSpacesInParametersAreSupported() throws Exception {
+        // emulates: bin/elasticsearch --node.name "'my node with spaces'" --pidfile "'/tmp/my pid.pid'"
+        BootstrapCLIParser parser = new BootstrapCLIParser(terminal);
+        registerProperties("es.pidfile", "es.my.param");
+
+        ExitStatus status = parser.execute("start", "--pidfile", "foo with space", "--my.param", "my awesome neighbour");
+        assertStatus(status, OK);
+        assertSystemProperty("es.pidfile", "foo with space");
+        assertSystemProperty("es.my.param", "my awesome neighbour");
+
+    }
+
+    private void registerProperties(String ... systemProperties) {
+        propertiesToClear.addAll(Arrays.asList(systemProperties));
+    }
+
+    private void assertSystemProperty(String name, String expectedValue) {
+        String msg = String.format(Locale.ROOT, "Expected property %s to be %s, terminal output was %s", name, expectedValue, terminal.getTerminalOutput());
+        assertThat(msg, System.getProperty(name), is(expectedValue));
+    }
+
+    private void assertStatus(ExitStatus status, ExitStatus expectedStatus) {
+        assertThat(String.format(Locale.ROOT, "Expected status to be [%s], but was [%s], terminal output was %s", expectedStatus, status, terminal.getTerminalOutput()), status, is(expectedStatus));
+    }
+
+    private void assertThatTerminalOutput(Matcher<String> matcher) {
+        assertThat(terminal.getTerminalOutput(), hasItem(matcher));
+    }
+}
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/MockDiskUsagesTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/MockDiskUsagesTests.java
@ -48,6 +48,7 @@ import static com.google.common.collect.Maps.newHashMap;
 import static org.elasticsearch.common.settings.Settings.settingsBuilder;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;

@ElasticsearchIntegrationTest.ClusterScope(scope = ElasticsearchIntegrationTest.Scope.TEST, numDataNodes = 0)
 public class MockDiskUsagesTests extends ElasticsearchIntegrationTest {
@ -59,13 +60,13 @@ public class MockDiskUsagesTests extends ElasticsearchIntegrationTest {
                        // Use the mock internal cluster info service, which has fake-able disk usages
                .put(ClusterModule.CLUSTER_SERVICE_IMPL, MockInternalClusterInfoService.class.getName())
                        // Update more frequently
-                .put(InternalClusterInfoService.INTERNAL_CLUSTER_INFO_UPDATE_INTERVAL, "2s")
+                .put(InternalClusterInfoService.INTERNAL_CLUSTER_INFO_UPDATE_INTERVAL, "1s")
                .build();
    }

    @Test
    //@TestLogging("org.elasticsearch.cluster:TRACE,org.elasticsearch.cluster.routing.allocation.decider:TRACE")
-    public void testRerouteOccursOnDiskpassingHighWatermark() throws Exception {
+    public void testRerouteOccursOnDiskPassingHighWatermark() throws Exception {
        List<String> nodes = internalCluster().startNodesAsync(3).get();

        // Wait for all 3 nodes to be up
@ -87,7 +88,7 @@ public class MockDiskUsagesTests extends ElasticsearchIntegrationTest {
        client().admin().cluster().prepareUpdateSettings().setTransientSettings(settingsBuilder()
                .put(DiskThresholdDecider.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK, randomFrom("20b", "80%"))
                .put(DiskThresholdDecider.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK, randomFrom("10b", "90%"))
-                .put(DiskThresholdDecider.CLUSTER_ROUTING_ALLOCATION_REROUTE_INTERVAL, "1s")).get();
+                .put(DiskThresholdDecider.CLUSTER_ROUTING_ALLOCATION_REROUTE_INTERVAL, "1ms")).get();

        // Create an index with 10 shards so we can check allocation for it
        prepareCreate("test").setSettings(settingsBuilder()
@ -106,7 +107,7 @@ public class MockDiskUsagesTests extends ElasticsearchIntegrationTest {
            }
        });

-        List<String> realNodeNames = newArrayList();
+        final List<String> realNodeNames = newArrayList();
        ClusterStateResponse resp = client().admin().cluster().prepareState().get();
        Iterator<RoutingNode> iter = resp.getState().getRoutingNodes().iterator();
        while (iter.hasNext()) {
@ -121,13 +122,14 @@ public class MockDiskUsagesTests extends ElasticsearchIntegrationTest {
        cis.setN2Usage(realNodeNames.get(1), new DiskUsage(nodes.get(1), "n2", 100, 50));
        cis.setN3Usage(realNodeNames.get(2), new DiskUsage(nodes.get(2), "n3", 100, 0)); // nothing free on node3

-        // Cluster info gathering interval is 2 seconds, give reroute 2 seconds to kick in
-        Thread.sleep(4000);
-
        // Retrieve the count of shards on each node
-        resp = client().admin().cluster().prepareState().get();
-        iter = resp.getState().getRoutingNodes().iterator();
-        Map<String, Integer> nodesToShardCount = newHashMap();
+        final Map<String, Integer> nodesToShardCount = newHashMap();
+
+        assertBusy(new Runnable() {
+            @Override
+            public void run() {
+                ClusterStateResponse resp = client().admin().cluster().prepareState().get();
+                Iterator<RoutingNode> iter = resp.getState().getRoutingNodes().iterator();
                while (iter.hasNext()) {
                    RoutingNode node = iter.next();
                    logger.info("--> node {} has {} shards",
@ -138,6 +140,33 @@ public class MockDiskUsagesTests extends ElasticsearchIntegrationTest {
                assertThat("node2 has 5 shards", nodesToShardCount.get(realNodeNames.get(1)), equalTo(5));
                assertThat("node3 has 0 shards", nodesToShardCount.get(realNodeNames.get(2)), equalTo(0));
            }
+        });
+
+        // Update the disk usages so one node is now back under the high watermark
+        cis.setN1Usage(realNodeNames.get(0), new DiskUsage(nodes.get(0), "n1", 100, 50));
+        cis.setN2Usage(realNodeNames.get(1), new DiskUsage(nodes.get(1), "n2", 100, 50));
+        cis.setN3Usage(realNodeNames.get(2), new DiskUsage(nodes.get(2), "n3", 100, 50)); // node3 has free space now
+
+        // Retrieve the count of shards on each node
+        nodesToShardCount.clear();
+
+        assertBusy(new Runnable() {
+            @Override
+            public void run() {
+                ClusterStateResponse resp = client().admin().cluster().prepareState().get();
+                Iterator<RoutingNode> iter = resp.getState().getRoutingNodes().iterator();
+                while (iter.hasNext()) {
+                    RoutingNode node = iter.next();
+                    logger.info("--> node {} has {} shards",
+                            node.nodeId(), resp.getState().getRoutingNodes().node(node.nodeId()).numberOfOwningShards());
+                    nodesToShardCount.put(node.nodeId(), resp.getState().getRoutingNodes().node(node.nodeId()).numberOfOwningShards());
+                }
+                assertThat("node1 has at least 3 shards", nodesToShardCount.get(realNodeNames.get(0)), greaterThanOrEqualTo(3));
+                assertThat("node2 has at least 3 shards", nodesToShardCount.get(realNodeNames.get(1)), greaterThanOrEqualTo(3));
+                assertThat("node3 has at least 3 shards", nodesToShardCount.get(realNodeNames.get(2)), greaterThanOrEqualTo(3));
+            }
+        });
+    }

    /** Create a fake NodeStats for the given node and usage */
    public static NodeStats makeStats(String nodeName, DiskUsage usage) {
--- a/core/src/test/java/org/elasticsearch/common/cli/CliToolTestCase.java
+++ b/core/src/test/java/org/elasticsearch/common/cli/CliToolTestCase.java
@ -21,6 +21,7 @@ package org.elasticsearch.common.cli;

 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.Streams;
 import org.elasticsearch.test.ElasticsearchTestCase;
 import org.junit.After;
 import org.junit.Before;
@ -33,6 +34,10 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.Locale;

+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.hasSize;
+
 /**
 *
 */
@ -149,4 +154,20 @@ public abstract class CliToolTestCase extends ElasticsearchTestCase {
            return terminalOutput;
        }
    }
+
+    public static void assertTerminalOutputContainsHelpFile(CliToolTestCase.CaptureOutputTerminal terminal, String classPath) throws IOException {
+        List<String> nonEmptyLines = new ArrayList<>();
+        for (String line : terminal.getTerminalOutput()) {
+            String originalPrintedLine = line.replaceAll(System.lineSeparator(), "");
+            if (com.google.common.base.Strings.isNullOrEmpty(originalPrintedLine)) {
+                nonEmptyLines.add(originalPrintedLine);
+            }
+        }
+        assertThat(nonEmptyLines, hasSize(greaterThan(0)));
+
+        String expectedDocs = Streams.copyToStringFromClasspath(classPath);
+        for (String nonEmptyLine : nonEmptyLines) {
+            assertThat(expectedDocs, containsString(nonEmptyLine.replaceAll(System.lineSeparator(), "")));
+        }
+    }
 }
--- a/core/src/test/java/org/elasticsearch/common/cli/CliToolTests.java
+++ b/core/src/test/java/org/elasticsearch/common/cli/CliToolTests.java
@ -33,6 +33,8 @@ import java.util.Map;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;

+import static org.elasticsearch.common.cli.CliTool.ExitStatus.OK;
+import static org.elasticsearch.common.cli.CliTool.ExitStatus.USAGE;
 import static org.elasticsearch.common.cli.CliToolConfig.Builder.cmd;
 import static org.hamcrest.Matchers.*;

@ -49,12 +51,12 @@ public class CliToolTests extends CliToolTestCase {
            @Override
            public CliTool.ExitStatus execute(Settings settings, Environment env) {
                executed.set(true);
-                return CliTool.ExitStatus.OK;
+                return OK;
            }
        };
        SingleCmdTool tool = new SingleCmdTool("tool", terminal, cmd);
-        int status = tool.execute();
-        assertStatus(status, CliTool.ExitStatus.OK);
+        CliTool.ExitStatus status = tool.execute();
+        assertStatus(status, OK);
        assertCommandHasBeenExecuted(executed);
    }

@ -70,7 +72,7 @@ public class CliToolTests extends CliToolTestCase {
            }
        };
        SingleCmdTool tool = new SingleCmdTool("tool", terminal, cmd);
-        int status = tool.execute();
+        CliTool.ExitStatus status = tool.execute();
        assertStatus(status, CliTool.ExitStatus.USAGE);
        assertCommandHasBeenExecuted(executed);
    }
@ -87,7 +89,7 @@ public class CliToolTests extends CliToolTestCase {
            }
        };
        SingleCmdTool tool = new SingleCmdTool("tool", terminal, cmd);
-        int status = tool.execute();
+        CliTool.ExitStatus status = tool.execute();
        assertStatus(status, CliTool.ExitStatus.IO_ERROR);
        assertCommandHasBeenExecuted(executed);
    }
@ -104,7 +106,7 @@ public class CliToolTests extends CliToolTestCase {
            }
        };
        SingleCmdTool tool = new SingleCmdTool("tool", terminal, cmd);
-        int status = tool.execute();
+        CliTool.ExitStatus status = tool.execute();
        assertStatus(status, CliTool.ExitStatus.CODE_ERROR);
        assertCommandHasBeenExecuted(executed);
    }
@ -124,14 +126,14 @@ public class CliToolTests extends CliToolTestCase {
                @Override
                public CliTool.ExitStatus execute(Settings settings, Environment env) throws Exception {
                    executed[index].set(true);
-                    return CliTool.ExitStatus.OK;
+                    return OK;
                }
            };
        }
        MultiCmdTool tool = new MultiCmdTool("tool", terminal, cmds);
        int cmdIndex = randomIntBetween(0, count-1);
-        int status = tool.execute("cmd" + cmdIndex);
-        assertThat(status, is(CliTool.ExitStatus.OK.status()));
+        CliTool.ExitStatus status = tool.execute("cmd" + cmdIndex);
+        assertThat(status, is(OK));
        for (int i = 0; i < executed.length; i++) {
            assertThat(executed[i].get(), is(i == cmdIndex));
        }
@ -152,13 +154,13 @@ public class CliToolTests extends CliToolTestCase {
                @Override
                public CliTool.ExitStatus execute(Settings settings, Environment env) throws Exception {
                    executed[index].set(true);
-                    return CliTool.ExitStatus.OK;
+                    return OK;
                }
            };
        }
        MultiCmdTool tool = new MultiCmdTool("tool", terminal, cmds);
-        int status = tool.execute("cmd" + count); // "cmd" + count doesn't exist
-        assertThat(status, is(CliTool.ExitStatus.USAGE.status()));
+        CliTool.ExitStatus status = tool.execute("cmd" + count); // "cmd" + count doesn't exist
+        assertThat(status, is(CliTool.ExitStatus.USAGE));
        for (int i = 0; i < executed.length; i++) {
            assertThat(executed[i].get(), is(false));
        }
@ -176,8 +178,8 @@ public class CliToolTests extends CliToolTestCase {
            }
        };
        SingleCmdTool tool = new SingleCmdTool("tool", terminal, cmd);
-        int status = tool.execute(args("-h"));
-        assertStatus(status, CliTool.ExitStatus.OK);
+        CliTool.ExitStatus status = tool.execute(args("-h"));
+        assertStatus(status, CliTool.ExitStatus.OK_AND_EXIT);
        assertThat(terminal.getTerminalOutput(), hasSize(3));
        assertThat(terminal.getTerminalOutput(), hasItem(containsString("cmd1 help")));
    }
@ -189,18 +191,18 @@ public class CliToolTests extends CliToolTestCase {
        cmds[0] = new NamedCommand("cmd0", terminal) {
            @Override
            public CliTool.ExitStatus execute(Settings settings, Environment env) throws Exception {
-                return CliTool.ExitStatus.OK;
+                return OK;
            }
        };
        cmds[1] = new NamedCommand("cmd1", terminal) {
            @Override
            public CliTool.ExitStatus execute(Settings settings, Environment env) throws Exception {
-                return CliTool.ExitStatus.OK;
+                return OK;
            }
        };
        MultiCmdTool tool = new MultiCmdTool("tool", terminal, cmds);
-        int status = tool.execute(args("-h"));
-        assertStatus(status, CliTool.ExitStatus.OK);
+        CliTool.ExitStatus status = tool.execute(args("-h"));
+        assertStatus(status, CliTool.ExitStatus.OK_AND_EXIT);
        assertThat(terminal.getTerminalOutput(), hasSize(3));
        assertThat(terminal.getTerminalOutput(), hasItem(containsString("tool help")));
    }
@ -212,18 +214,18 @@ public class CliToolTests extends CliToolTestCase {
        cmds[0] = new NamedCommand("cmd0", terminal) {
            @Override
            public CliTool.ExitStatus execute(Settings settings, Environment env) throws Exception {
-                return CliTool.ExitStatus.OK;
+                return OK;
            }
        };
        cmds[1] = new NamedCommand("cmd1", terminal) {
            @Override
            public CliTool.ExitStatus execute(Settings settings, Environment env) throws Exception {
-                return CliTool.ExitStatus.OK;
+                return OK;
            }
        };
        MultiCmdTool tool = new MultiCmdTool("tool", terminal, cmds);
-        int status = tool.execute(args("cmd1 -h"));
-        assertStatus(status, CliTool.ExitStatus.OK);
+        CliTool.ExitStatus status = tool.execute(args("cmd1 -h"));
+        assertStatus(status, CliTool.ExitStatus.OK_AND_EXIT);
        assertThat(terminal.getTerminalOutput(), hasSize(3));
        assertThat(terminal.getTerminalOutput(), hasItem(containsString("cmd1 help")));
    }
@ -264,7 +266,7 @@ public class CliToolTests extends CliToolTestCase {
            @Override
            public CliTool.ExitStatus execute(Settings settings, Environment env) {
                executed.set(true);
-                return CliTool.ExitStatus.OK;
+                return OK;
            }
        };
        SingleCmdTool tool = new SingleCmdTool("tool", terminal, cmd);
@ -298,7 +300,7 @@ public class CliToolTests extends CliToolTestCase {
            public CliTool.ExitStatus execute(Settings settings, Environment env) {
                promptedSecretValue.set(settings.get("foo.password"));
                promptedTextValue.set(settings.get("replace"));
-                return CliTool.ExitStatus.OK;
+                return OK;
            }
        };

@ -316,8 +318,44 @@ public class CliToolTests extends CliToolTestCase {
        assertThat(promptedTextValue.get(), is("replaced"));
    }

-    private void assertStatus(int status, CliTool.ExitStatus expectedStatus) {
-        assertThat(status, is(expectedStatus.status()));
+    @Test
+    public void testStopAtNonOptionParsing() throws Exception {
+        final CliToolConfig.Cmd lenientCommand = cmd("lenient", CliTool.Command.Exit.class).stopAtNonOption(true).build();
+        final CliToolConfig.Cmd strictCommand = cmd("strict", CliTool.Command.Exit.class).stopAtNonOption(false).build();
+        final CliToolConfig config = CliToolConfig.config("elasticsearch", CliTool.class).cmds(lenientCommand, strictCommand).build();
+
+        final CaptureOutputTerminal terminal = new CaptureOutputTerminal();
+        final CliTool cliTool = new CliTool(config, terminal) {
+            @Override
+            protected Command parse(String cmdName, CommandLine cli) throws Exception {
+                return new NamedCommand(cmdName, terminal) {
+                    @Override
+                    public ExitStatus execute(Settings settings, Environment env) throws Exception {
+                        return OK;
+                    }
+                };
+            }
+        };
+
+        // known parameters, no error
+        assertStatus(cliTool.execute(args("lenient --verbose")), OK);
+        assertStatus(cliTool.execute(args("lenient -v")), OK);
+
+        // unknown parameters, no error
+        assertStatus(cliTool.execute(args("lenient --unknown")), OK);
+        assertStatus(cliTool.execute(args("lenient -u")), OK);
+
+        // unknown parameters, error
+        assertStatus(cliTool.execute(args("strict --unknown")), USAGE);
+        assertThat(terminal.getTerminalOutput(), hasItem(containsString("Unrecognized option: --unknown")));
+
+        terminal.getTerminalOutput().clear();
+        assertStatus(cliTool.execute(args("strict -u")), USAGE);
+        assertThat(terminal.getTerminalOutput(), hasItem(containsString("Unrecognized option: -u")));
+    }
+
+    private void assertStatus(CliTool.ExitStatus status, CliTool.ExitStatus expectedStatus) {
+        assertThat(status, is(expectedStatus));
    }

    private void assertCommandHasBeenExecuted(AtomicReference<Boolean> executed) {
--- a/core/src/test/java/org/elasticsearch/plugins/PluginManagerCliTests.java
+++ b/core/src/test/java/org/elasticsearch/plugins/PluginManagerCliTests.java
@ -19,16 +19,12 @@

 package org.elasticsearch.plugins;

-import com.google.common.base.Strings;
 import org.elasticsearch.common.cli.CliToolTestCase;
-import org.elasticsearch.common.io.Streams;
 import org.junit.Test;

 import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;

-import static org.elasticsearch.common.cli.CliTool.ExitStatus.OK;
+import static org.elasticsearch.common.cli.CliTool.ExitStatus.OK_AND_EXIT;
 import static org.hamcrest.Matchers.*;

 public class PluginManagerCliTests extends CliToolTestCase {
@ -36,38 +32,22 @@ public class PluginManagerCliTests extends CliToolTestCase {
    @Test
    public void testHelpWorks() throws IOException {
        CliToolTestCase.CaptureOutputTerminal terminal = new CliToolTestCase.CaptureOutputTerminal();
-        assertThat(new PluginManagerCliParser(terminal).execute(args("--help")), is(OK.status()));
-        assertHelp(terminal, "/org/elasticsearch/plugins/plugin.help");
+        assertThat(new PluginManagerCliParser(terminal).execute(args("--help")), is(OK_AND_EXIT));
+        assertTerminalOutputContainsHelpFile(terminal, "/org/elasticsearch/plugins/plugin.help");

        terminal.getTerminalOutput().clear();
-        assertThat(new PluginManagerCliParser(terminal).execute(args("install -h")), is(OK.status()));
-        assertHelp(terminal, "/org/elasticsearch/plugins/plugin-install.help");
+        assertThat(new PluginManagerCliParser(terminal).execute(args("install -h")), is(OK_AND_EXIT));
+        assertTerminalOutputContainsHelpFile(terminal, "/org/elasticsearch/plugins/plugin-install.help");
        for (String plugin : PluginManager.OFFICIAL_PLUGINS) {
            assertThat(terminal.getTerminalOutput(), hasItem(containsString(plugin)));
        }

        terminal.getTerminalOutput().clear();
-        assertThat(new PluginManagerCliParser(terminal).execute(args("remove --help")), is(OK.status()));
-        assertHelp(terminal, "/org/elasticsearch/plugins/plugin-remove.help");
+        assertThat(new PluginManagerCliParser(terminal).execute(args("remove --help")), is(OK_AND_EXIT));
+        assertTerminalOutputContainsHelpFile(terminal, "/org/elasticsearch/plugins/plugin-remove.help");

        terminal.getTerminalOutput().clear();
-        assertThat(new PluginManagerCliParser(terminal).execute(args("list -h")), is(OK.status()));
-        assertHelp(terminal, "/org/elasticsearch/plugins/plugin-list.help");
-    }
-
-    private void assertHelp(CliToolTestCase.CaptureOutputTerminal terminal, String classPath) throws IOException {
-        List<String> nonEmptyLines = new ArrayList<>();
-        for (String line : terminal.getTerminalOutput()) {
-            String originalPrintedLine = line.replaceAll(System.lineSeparator(), "");
-            if (Strings.isNullOrEmpty(originalPrintedLine)) {
-                nonEmptyLines.add(originalPrintedLine);
-            }
-        }
-        assertThat(nonEmptyLines, hasSize(greaterThan(0)));
-
-        String expectedDocs = Streams.copyToStringFromClasspath(classPath);
-        for (String nonEmptyLine : nonEmptyLines) {
-            assertThat(expectedDocs, containsString(nonEmptyLine.replaceAll(System.lineSeparator(), "")));
-        }
+        assertThat(new PluginManagerCliParser(terminal).execute(args("list -h")), is(OK_AND_EXIT));
+        assertTerminalOutputContainsHelpFile(terminal, "/org/elasticsearch/plugins/plugin-list.help");
    }
 }
--- a/core/src/test/java/org/elasticsearch/plugins/PluginManagerTests.java
+++ b/core/src/test/java/org/elasticsearch/plugins/PluginManagerTests.java
@ -21,7 +21,7 @@ package org.elasticsearch.plugins;
 import org.apache.http.impl.client.HttpClients;
 import org.apache.lucene.util.LuceneTestCase;
 import org.elasticsearch.Version;
-import org.elasticsearch.common.cli.CliTool;
+import org.elasticsearch.common.cli.CliTool.ExitStatus;
 import org.elasticsearch.common.cli.CliToolTestCase.CaptureOutputTerminal;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.settings.Settings;
@ -54,11 +54,11 @@ import static org.elasticsearch.common.cli.CliTool.ExitStatus.USAGE;
 import static org.elasticsearch.common.cli.CliToolTestCase.args;
 import static org.elasticsearch.common.io.FileSystemUtilsTests.assertFileContent;
 import static org.elasticsearch.common.settings.Settings.settingsBuilder;
+import static org.elasticsearch.plugins.PluginInfoTests.writeProperties;
 import static org.elasticsearch.test.ElasticsearchIntegrationTest.Scope;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertDirectoryExists;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertFileExists;
 import static org.hamcrest.Matchers.*;
-import static org.elasticsearch.plugins.PluginInfoTests.writeProperties;

@ClusterScope(scope = Scope.TEST, numDataNodes = 0, transportClientRatio = 0.0)
@LuceneTestCase.SuppressFileSystems("*") // TODO: clean up this test to allow extra files
@ -434,15 +434,13 @@ public class PluginManagerTests extends ElasticsearchIntegrationTest {

    @Test
    public void testRemovePlugin_NullName_ThrowsException() throws IOException {
-        int status = new PluginManagerCliParser(terminal).execute(args("remove "));
-        assertThat("Terminal output was: " + terminal.getTerminalOutput(), status, is(USAGE.status()));
+        assertStatus("remove ", USAGE);
    }

    @Test
    public void testRemovePluginWithURLForm() throws Exception {
-        int status = new PluginManagerCliParser(terminal).execute(args("remove file://whatever"));
+        assertStatus("remove file://whatever", USAGE);
        assertThat(terminal.getTerminalOutput(), hasItem(containsString("Illegal plugin name")));
-        assertThat("Terminal output was: " + terminal.getTerminalOutput(), status, is(USAGE.status()));
    }

    @Test
@ -488,12 +486,12 @@ public class PluginManagerTests extends ElasticsearchIntegrationTest {
    }

    private void assertStatusOk(String command) {
-        assertStatus(command, CliTool.ExitStatus.OK);
+        assertStatus(command, ExitStatus.OK);
    }

-    private void assertStatus(String command, CliTool.ExitStatus exitStatus) {
-        int status = new PluginManagerCliParser(terminal).execute(args(command));
-        assertThat("Terminal output was: " + terminal.getTerminalOutput(), status, is(exitStatus.status()));
+    private void assertStatus(String command, ExitStatus exitStatus) {
+        ExitStatus status = new PluginManagerCliParser(terminal).execute(args(command));
+        assertThat("Terminal output was: " + terminal.getTerminalOutput(), status, is(exitStatus));
    }

    private void assertThatPluginIsListed(String pluginName) {
--- a/dev-tools/src/main/resources/ant/integration-tests.xml
+++ b/dev-tools/src/main/resources/ant/integration-tests.xml
@ -183,4 +183,85 @@
    <delete file="${integ.pidfile}"/>
  </target>

+  <!-- distribution tests: .tar.gz -->
+
+  <target name="setup-workspace-tar" depends="stop-external-cluster" unless="${shouldskip}">
+    <sequential>
+      <delete dir="${integ.scratch}"/>
+      <untar src="${project.build.directory}/releases/elasticsearch-${project.version}.tar.gz" 
+             dest="${integ.scratch}"
+             compression="gzip"/>
+    </sequential>
+  </target>
+
+  <target name="start-external-cluster-tar" depends="setup-workspace-tar" unless="${shouldskip}">
+    <startup-elasticsearch/>
+  </target>
+
+  <!-- distribution tests: .deb -->
+
+  <target name="setup-workspace-deb" depends="stop-external-cluster" unless="${shouldskip}">
+    <sequential>
+      <delete dir="${integ.scratch}"/>
+      <mkdir dir="${integ.scratch}/deb-extracted"/>
+      <local name="debfile"/>
+      <property name="debfile" location="${project.build.directory}/releases/elasticsearch-${project.version}.deb"/>
+      <!-- print some basic package info -->
+      <exec executable="dpkg-deb" failonerror="true" taskname="deb-info">
+         <arg value="-I"/>
+         <arg value="${debfile}"/> 
+      </exec>
+      <!-- extract contents from .deb package -->
+      <exec executable="dpkg-deb" failonerror="true">
+         <arg value="-x"/> 
+         <arg value="${debfile}"/> 
+         <arg value="${integ.scratch}/deb-extracted"/> 
+      </exec>
+    </sequential>
+  </target>
+
+  <target name="start-external-cluster-deb" depends="setup-workspace-deb" unless="${shouldskip}">
+    <startup-elasticsearch home="${integ.scratch}/deb-extracted/usr/share/elasticsearch/"/>
+  </target>
+
+  <!-- distribution tests: .rpm -->
+  <target name="setup-workspace-rpm" depends="stop-external-cluster" unless="${shouldskip}">
+    <sequential>
+      <delete dir="${integ.scratch}"/>
+      <!-- use full paths with paranoia, we will be doing relocations -->
+      <local name="rpm.file"/>
+      <local name="rpm.database"/>
+      <local name="rpm.extracted"/>
+      <property name="rpm.file" location="${project.build.directory}/releases/elasticsearch-${project.version}.rpm"/>
+      <property name="rpm.database" location="${integ.scratch}/rpm-database"/>
+      <property name="rpm.extracted" location="${integ.scratch}/rpm-extracted"/>
+      <mkdir dir="${rpm.database}"/>
+      <mkdir dir="${rpm.extracted}"/>
+      <!-- print some basic package info -->
+      <exec executable="rpm" failonerror="true" taskname="rpm-info">
+         <arg value="-q"/>
+         <arg value="-i"/>
+         <arg value="-p"/>
+         <arg value="${rpm.file}"/> 
+      </exec>
+      <!-- extract contents from .rpm package -->
+      <exec executable="rpm" failonerror="true" taskname="rpm">
+         <arg value="--dbpath"/>
+         <arg value="${rpm.database}"/>
+         <arg value="--badreloc"/>
+         <arg value="--relocate"/>
+         <arg value="/=${rpm.extracted}"/>
+         <arg value="--nodeps"/> 
+         <arg value="--noscripts"/> 
+         <arg value="--notriggers"/> 
+         <arg value="-i"/>
+         <arg value="${rpm.file}"/> 
+      </exec>
+    </sequential>
+  </target>
+
+  <target name="start-external-cluster-rpm" depends="setup-workspace-rpm" unless="${shouldskip}">
+    <startup-elasticsearch home="${integ.scratch}/rpm-extracted/usr/share/elasticsearch/"/>
+  </target>
+
 </project>
--- a/distribution/deb/pom.xml
+++ b/distribution/deb/pom.xml
@ -10,11 +10,9 @@
    </parent>

    <artifactId>elasticsearch-deb</artifactId>
-    <packaging>deb</packaging>
    <name>Elasticsearch DEB Distribution</name>

    <properties>
-        <skip.integ.tests>true</skip.integ.tests>
        <deb.sign>false</deb.sign>
        <deb.sign.method>dpkg-sig</deb.sign.method>
    </properties>
@ -273,8 +271,49 @@
                            </target>
                        </configuration>
                    </execution>
+                    <!-- start up external cluster -->
+                    <execution>
+                        <id>integ-setup</id>
+                        <phase>pre-integration-test</phase>
+                        <goals>
+                            <goal>run</goal>
+                        </goals>
+                        <configuration>
+                            <skip>${skip.integ.tests}</skip>
+                            <target>
+                                <ant antfile="${elasticsearch.integ.antfile}" target="start-external-cluster-deb"/>
+                            </target>
+                        </configuration>
+                    </execution>
+                    <!-- shut down external cluster -->
+                    <execution>
+                        <id>integ-teardown</id>
+                        <phase>post-integration-test</phase>
+                        <goals>
+                            <goal>run</goal>
+                        </goals>
+                        <configuration>
+                            <skip>${skip.integ.tests}</skip>
+                            <target>
+                                <ant antfile="${elasticsearch.integ.antfile}" target="stop-external-cluster"/>
+                            </target>
+                        </configuration>
+                    </execution>
                </executions>
            </plugin>
        </plugins>
    </build>
+
+    <profiles>
+       <!-- we run integration test with real dpkg utils, you must have them -->
+       <profile>
+           <id>has_dpkg</id>
+           <activation>
+              <file><missing>/usr/bin/dpkg-deb</missing></file>
+           </activation>
+           <properties>
+               <skip.integ.tests>true</skip.integ.tests>
+           </properties>
+       </profile>
+    </profiles>
 </project>
--- a/distribution/deb/src/test/java/org/elasticsearch/test/rest/RestIT.java
+++ b/distribution/deb/src/test/java/org/elasticsearch/test/rest/RestIT.java
@ -0,0 +1,38 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.test.rest;
+
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+
+import org.elasticsearch.test.rest.parser.RestTestParseException;
+
+import java.io.IOException;
+
+/** Rest integration test. runs against external cluster in 'mvn verify' */
+public class RestIT extends ElasticsearchRestTestCase {
+    public RestIT(RestTestCandidate testCandidate) {
+        super(testCandidate);
+    }
+    // we run them all sequentially: start simple!
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() throws IOException, RestTestParseException {
+        return createParameters(0, 1);
+    }
+}
--- a/distribution/licenses/commons-cli-1.2.jar.sha1
+++ b/distribution/licenses/commons-cli-1.2.jar.sha1
@ -1 +0,0 @@
-2bf96b7aa8b611c177d329452af1dc933e14501c
--- a/distribution/licenses/commons-cli-1.3.1.jar.sha1
+++ b/distribution/licenses/commons-cli-1.3.1.jar.sha1
@ -0,0 +1 @@
+1303efbc4b181e5a58bf2e967dc156a3132b97c0
--- a/distribution/pom.xml
+++ b/distribution/pom.xml
@ -15,7 +15,6 @@
    <name>Elasticsearch Distribution</name>

    <properties>
-        <skipTests>true</skipTests>
        <!-- Properties used for building RPM & DEB packages (see common/packaging.properties) -->
        <packaging.elasticsearch.home.dir>/usr/share/elasticsearch</packaging.elasticsearch.home.dir>
        <packaging.elasticsearch.bin.dir>/usr/share/elasticsearch/bin</packaging.elasticsearch.bin.dir>
@ -32,6 +31,9 @@

        <!-- rpmbuild location : default to /usr/bin/rpmbuild -->
        <packaging.rpm.rpmbuild>/usr/bin/rpmbuild</packaging.rpm.rpmbuild>
+
+        <!-- we expect packaging formats to have integration tests, but not unit tests -->
+        <skip.unit.tests>true</skip.unit.tests>
    </properties>

    <dependencies>
@ -90,6 +92,30 @@
            <scope>test</scope>
        </dependency>

+        <dependency>
+            <groupId>org.hamcrest</groupId>
+            <artifactId>hamcrest-all</artifactId>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.lucene</groupId>
+            <artifactId>lucene-test-framework</artifactId>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.elasticsearch</groupId>
+            <artifactId>elasticsearch</artifactId>
+            <scope>test</scope>
+            <type>test-jar</type>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.httpcomponents</groupId>
+            <artifactId>httpclient</artifactId>
+            <scope>test</scope>
+        </dependency>
    </dependencies>

    <build>
--- a/distribution/rpm/pom.xml
+++ b/distribution/rpm/pom.xml
@ -14,7 +14,6 @@
    <packaging>rpm</packaging>

    <properties>
-        <skip.integ.tests>true</skip.integ.tests>
    </properties>

    <build>
@ -332,6 +331,34 @@
                            </target>
                        </configuration>
                    </execution>
+                    <!-- start up external cluster -->
+                    <execution>
+                        <id>integ-setup</id>
+                        <phase>pre-integration-test</phase>
+                        <goals>
+                            <goal>run</goal>
+                        </goals>
+                        <configuration>
+                            <skip>${skip.integ.tests}</skip>
+                            <target>
+                                <ant antfile="${elasticsearch.integ.antfile}" target="start-external-cluster-rpm"/>
+                            </target>
+                        </configuration>
+                    </execution>
+                    <!-- shut down external cluster -->
+                    <execution>
+                        <id>integ-teardown</id>
+                        <phase>post-integration-test</phase>
+                        <goals>
+                            <goal>run</goal>
+                        </goals>
+                        <configuration>
+                            <skip>${skip.integ.tests}</skip>
+                            <target>
+                                <ant antfile="${elasticsearch.integ.antfile}" target="stop-external-cluster"/>
+                            </target>
+                        </configuration>
+                    </execution>
                </executions>
            </plugin>
        </plugins>
--- a/distribution/rpm/src/test/java/org/elasticsearch/test/rest/RestIT.java
+++ b/distribution/rpm/src/test/java/org/elasticsearch/test/rest/RestIT.java
@ -0,0 +1,38 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.test.rest;
+
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+
+import org.elasticsearch.test.rest.parser.RestTestParseException;
+
+import java.io.IOException;
+
+/** Rest integration test. runs against external cluster in 'mvn verify' */
+public class RestIT extends ElasticsearchRestTestCase {
+    public RestIT(RestTestCandidate testCandidate) {
+        super(testCandidate);
+    }
+    // we run them all sequentially: start simple!
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() throws IOException, RestTestParseException {
+        return createParameters(0, 1);
+    }
+}
--- a/distribution/src/main/resources/bin/elasticsearch
+++ b/distribution/src/main/resources/bin/elasticsearch
@ -1,16 +1,5 @@
 #!/bin/sh

-# OPTIONS:
-#    -d            daemonize (run in background)
-#    -p pidfile    write PID to <pidfile>
-#    -h
-#    --help        print command line options
-#    -v            print elasticsearch version, then exit
-#    -D prop       set JAVA system property
-#    -X prop       set non-standard JAVA system property
-#   --prop=val
-#   --prop val     set elasticsearch property (i.e.  -Des.<prop>=<val>)
-
 # CONTROLLING STARTUP:
 #
 # This script relies on few environment variables to determine startup
@ -132,120 +121,16 @@ case `uname` in
    ;;
 esac

-launch_service()
-{
-    pidpath=$1
-    daemonized=$2
-    props=$3
-    es_parms="-Delasticsearch"
-
-    if [ "x$pidpath" != "x" ]; then
-        es_parms="$es_parms -Des.pidfile=$pidpath"
-    fi
-
-    # Make sure we dont use any predefined locale, as we check some exception message strings and rely on english language
-    # As those strings are created by the OS, they are dependant on the configured locale
-    LANG=en_US.UTF-8
-    LC_ALL=en_US.UTF-8
-
 export HOSTNAME=`hostname -s`

-    # The es-foreground option will tell Elasticsearch not to close stdout/stderr, but it's up to us not to daemonize.
-    if [ "x$daemonized" = "x" ]; then
-        es_parms="$es_parms -Des.foreground=yes"
-        eval exec "$JAVA" $JAVA_OPTS $ES_JAVA_OPTS $es_parms "\"-Des.path.home=$ES_HOME\"" -cp "\"$ES_CLASSPATH\"" $props \
-                org.elasticsearch.bootstrap.Elasticsearch
-        # exec without running it in the background, makes it replace this shell, we'll never get here...
-        # no need to return something
+# manual parsing to find out, if process should be detached
+daemonized=`echo $* | grep -E -- '(^-d |-d$| -d |--daemonize$|--daemonize )'`
+if [ -z "$daemonized" ] ; then
+    eval exec "$JAVA" $JAVA_OPTS $ES_JAVA_OPTS "\"-Des.path.home=$ES_HOME\"" -cp "\"$ES_CLASSPATH\"" \
+          org.elasticsearch.bootstrap.Elasticsearch start $*
 else
-        # Startup Elasticsearch, background it, and write the pid.
-        eval exec "$JAVA" $JAVA_OPTS $ES_JAVA_OPTS $es_parms "\"-Des.path.home=$ES_HOME\"" -cp "\"$ES_CLASSPATH\"" $props \
-                    org.elasticsearch.bootstrap.Elasticsearch <&- &
-        return $?
+    eval exec "$JAVA" $JAVA_OPTS $ES_JAVA_OPTS "\"-Des.path.home=$ES_HOME\"" -cp "\"$ES_CLASSPATH\"" \
+          org.elasticsearch.bootstrap.Elasticsearch start $* <&- &
 fi
-}
-
-# Print command line usage / help
-usage() {
-    echo "Usage: $0 [-vdh] [-p pidfile] [-D prop] [-X prop]"
-    echo "Start elasticsearch."
-    echo "    -d            daemonize (run in background)"
-    echo "    -p pidfile    write PID to <pidfile>"
-    echo "    -h"
-    echo "    --help        print command line options"
-    echo "    -v            print elasticsearch version, then exit"
-    echo "    -D prop       set JAVA system property"
-    echo "    -X prop       set non-standard JAVA system property"
-    echo "   --prop=val"
-    echo "   --prop val     set elasticsearch property (i.e. -Des.<prop>=<val>)"
-}
-
-# Parse any long getopt options and put them into properties before calling getopt below
-# Be dash compatible to make sure running under ubuntu works
-ARGV=""
-while [ $# -gt 0 ]
-do
-    case $1 in
-      --help) ARGV="$ARGV -h"; shift;;
-      --*=*) properties="$properties -Des.${1#--}"
-           shift 1
-           ;;
-      --*) [ $# -le 1 ] && {
-                echo "Option requires an argument: '$1'."
-                shift
-                continue
-            }
-           properties="$properties -Des.${1#--}=$2"
-           shift 2
-           ;;
-      *) ARGV="$ARGV $1" ; shift
-    esac
-done
-
-# Parse any command line options.
-args=`getopt vdhp:D:X: $ARGV`
-eval set -- "$args"
-
-while true; do
-    case $1 in
-        -v)
-            eval "$JAVA" $JAVA_OPTS $ES_JAVA_OPTS $es_parms "\"-Des.path.home=$ES_HOME\"" -cp "\"$ES_CLASSPATH\"" $props \
-                    org.elasticsearch.Version
-            exit 0
-        ;;
-        -p)
-            pidfile="$2"
-            shift 2
-        ;;
-        -d)
-            daemonized="yes"
-            shift
-        ;;
-        -h)
-            usage
-            exit 0
-        ;;
-        -D)
-            properties="$properties -D$2"
-            shift 2
-        ;;
-        -X)
-            properties="$properties -X$2"
-            shift 2
-        ;;
-        --)
-            shift
-            break
-        ;;
-        *)
-            echo "Error parsing argument $1!" >&2
-            usage
-            exit 1
-        ;;
-    esac
-done
-
-# Start up the service
-launch_service "$pidfile" "$daemonized" "$properties"

 exit $?
--- a/distribution/src/main/resources/bin/elasticsearch.bat
+++ b/distribution/src/main/resources/bin/elasticsearch.bat
@ -43,6 +43,6 @@ IF ERRORLEVEL 1 (
 	EXIT /B %ERRORLEVEL%
 )

-"%JAVA_HOME%\bin\java" %JAVA_OPTS% %ES_JAVA_OPTS% %ES_PARAMS% !newparams! -cp "%ES_CLASSPATH%" "org.elasticsearch.bootstrap.Elasticsearch"
+"%JAVA_HOME%\bin\java" %JAVA_OPTS% %ES_JAVA_OPTS% %ES_PARAMS% !newparams! -cp "%ES_CLASSPATH%" "org.elasticsearch.bootstrap.Elasticsearch" start

 ENDLOCAL
--- a/distribution/tar/pom.xml
+++ b/distribution/tar/pom.xml
@ -13,7 +13,6 @@
    <name>Elasticsearch TAR Distribution</name>

    <properties>
-        <skip.integ.tests>true</skip.integ.tests>
    </properties>

    <build>
@ -68,6 +67,35 @@
                            </target>
                        </configuration>
                    </execution>
+                    <!-- integration tests -->
+                    <!-- start up external cluster -->
+                    <execution>
+                        <id>integ-setup</id>
+                        <phase>pre-integration-test</phase>
+                        <goals>
+                            <goal>run</goal>
+                        </goals>
+                        <configuration>
+                            <skip>${skip.integ.tests}</skip>
+                            <target>
+                                <ant antfile="${elasticsearch.integ.antfile}" target="start-external-cluster-tar"/>
+                            </target>
+                        </configuration>
+                    </execution>
+                    <!-- shut down external cluster -->
+                    <execution>
+                        <id>integ-teardown</id>
+                        <phase>post-integration-test</phase>
+                        <goals>
+                            <goal>run</goal>
+                        </goals>
+                        <configuration>
+                            <skip>${skip.integ.tests}</skip>
+                            <target>
+                                <ant antfile="${elasticsearch.integ.antfile}" target="stop-external-cluster"/>
+                            </target>
+                        </configuration>
+                    </execution>
                </executions>
            </plugin>
        </plugins>
--- a/distribution/tar/src/test/java/org/elasticsearch/test/rest/RestIT.java
+++ b/distribution/tar/src/test/java/org/elasticsearch/test/rest/RestIT.java
@ -0,0 +1,38 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.test.rest;
+
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+
+import org.elasticsearch.test.rest.parser.RestTestParseException;
+
+import java.io.IOException;
+
+/** Rest integration test. runs against external cluster in 'mvn verify' */
+public class RestIT extends ElasticsearchRestTestCase {
+    public RestIT(RestTestCandidate testCandidate) {
+        super(testCandidate);
+    }
+    // we run them all sequentially: start simple!
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() throws IOException, RestTestParseException {
+        return createParameters(0, 1);
+    }
+}
--- a/distribution/zip/pom.xml
+++ b/distribution/zip/pom.xml
@ -16,35 +16,6 @@
        <skip.integ.tests>false</skip.integ.tests>
    </properties>

-    <dependencies>
-        <dependency>
-            <groupId>org.hamcrest</groupId>
-            <artifactId>hamcrest-all</artifactId>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>com.carrotsearch.randomizedtesting</groupId>
-            <artifactId>randomizedtesting-runner</artifactId>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.lucene</groupId>
-            <artifactId>lucene-test-framework</artifactId>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>org.elasticsearch</groupId>
-            <artifactId>elasticsearch</artifactId>
-            <scope>test</scope>
-            <type>test-jar</type>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.httpcomponents</groupId>
-            <artifactId>httpclient</artifactId>
-            <scope>test</scope>
-        </dependency>
-    </dependencies>
-
    <build>
        <filters>
            <filter>${project.basedir}/../src/main/packaging/packaging.properties</filter>
@ -119,6 +90,7 @@
                            <goal>run</goal>
                        </goals>
                        <configuration>
+                            <skip>${skip.integ.tests}</skip>
                            <target>
                                <ant antfile="${elasticsearch.integ.antfile}" target="start-external-cluster"/>
                            </target>
@ -132,6 +104,7 @@
                            <goal>run</goal>
                        </goals>
                        <configuration>
+                            <skip>${skip.integ.tests}</skip>
                            <target>
                                <ant antfile="${elasticsearch.integ.antfile}" target="stop-external-cluster"/>
                            </target>
--- a/docs/java-api/client.asciidoc
+++ b/docs/java-api/client.asciidoc
@ -90,7 +90,14 @@ import static org.elasticsearch.node.NodeBuilder.*;

 // on startup

-Node node = nodeBuilder().client(true).node();
+// Embedded node clients behave just like standalone nodes,
+// which means that they will leave the HTTP port open!
+Node node =
+    nodeBuilder()
+        .settings(Settings.settingsBuilder().put("http.enabled", false))
+        .client(true)
+    .node();
+
 Client client = node.client();

 // on shutdown
@ -120,11 +127,21 @@ Client client = node.client();
 node.close();
 --------------------------------------------------

+[[node-client-downsides]]
+==== Node Client Downsides
+
+Embedding a node client into your application is the easiest way to connect
+to an Elasticsearch cluster, but it carries some downsides.
+
+- Frequently starting and stopping one or more node clients creates unnecessary
+noise across the cluster.
+- Embedded node client will respond to outside requests, just like any other client.
+** You almost always want to disable HTTP for an _embedded_ node client.

 [[transport-client]]
 === Transport Client

-The `TransportClient` connects remotely to an elasticsearch cluster
+The `TransportClient` connects remotely to an Elasticsearch cluster
 using the transport module. It does not join the cluster, but simply
 gets one or more initial transport addresses and communicates with them
 in round robin fashion on each action (though most actions will probably
@ -183,4 +200,3 @@ from a node. Defaults to `5s`.
 |`client.transport.nodes_sampler_interval` |How often to sample / ping
 the nodes listed and connected. Defaults to `5s`.
 |=======================================================================
-
--- a/docs/reference/migration/migrate_2_0.asciidoc
+++ b/docs/reference/migration/migrate_2_0.asciidoc
@ -820,6 +820,13 @@ For the record, official plugins which can use this new simplified form are:
 * elasticsearch-lang-javascript
 * elasticsearch-lang-python

+=== `/bin/elasticsearch` version needs `-V` parameter
+
+Due to switching to elasticsearchs internal command line parsing
+infrastructure for the pluginmanager and the elasticsearch start up
+script, the `-v` parameter now stands for `--verbose`, where as `-V` or
+`--version` can be used to show the Elasticsearch version and exit.
+
 === Aliases

 Fields used in alias filters no longer have to exist in the mapping upon alias creation time. Alias filters are now
--- a/pom.xml
+++ b/pom.xml
@ -409,7 +409,7 @@
            <dependency>
                <groupId>commons-cli</groupId>
                <artifactId>commons-cli</artifactId>
-                <version>1.2</version>
+                <version>1.3.1</version>
            </dependency>

            <!-- END: dependencies that might be shaded -->
@ -1337,7 +1337,7 @@ org.eclipse.jdt.ui.text.custom_code_templates=<?xml version\="1.0" encoding\="UT
            <activation>
                <activeByDefault>false</activeByDefault>
                <file>
-                    <missing>${basedir}/src</missing>
+                    <missing>${basedir}/src/test/java</missing>
                </file>
            </activation>
            <properties>
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.nodeattrs/10_basic.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.nodeattrs/10_basic.yaml
@ -1,6 +1,10 @@
 ---
 "Test cat nodes attrs output":

+  - skip:
+      version:  "all"
+      reason:   "Waiting for #12558"
+
  - do:
      cat.nodeattrs:
        v: false