Add online plugin repository and a 'plugin' command to download them, closes #157

bin/plugin

@@ -0,0 +1,33 @@
+#!/bin/sh
+
+SCRIPT="$0"
+
+# SCRIPT may be an arbitrarily deep series of symlinks. Loop until we have the concrete path.
+while [ -h "$SCRIPT" ] ; do
+  ls=`ls -ld "$SCRIPT"`
+  # Drop everything prior to ->
+  link=`expr "$ls" : '.*-> \(.*\)$'`
+  if expr "$link" : '/.*' > /dev/null; then
+    SCRIPT="$link"
+  else
+    SCRIPT=`dirname "$SCRIPT"`/"$link"
+  fi
+done
+
+# determine elasticsearch home
+ES_HOME=`dirname "$SCRIPT"`/..
+
+# make ELASTICSEARCH_HOME absolute
+ES_HOME=`cd $ES_HOME; pwd`
+
+
+if [ -x $JAVA_HOME/bin/java ]; then
+    JAVA=$JAVA_HOME/bin/java
+else
+    JAVA=`which java`
+fi
+
+CLASSPATH=$CLASSPATH:$ES_HOME/lib/*
+
+$JAVA -Delasticsearch -Des.path.home=$ES_HOME -cp $CLASSPATH org.elasticsearch.plugins.PluginManager $*
+  

bin/plugin.bat

@@ -0,0 +1,25 @@
+@echo off
+
+SETLOCAL
+
+if NOT DEFINED JAVA_HOME goto err
+
+set SCRIPT_DIR=%~dp0
+for %%I in ("%SCRIPT_DIR%..") do set ES_HOME=%%~dpfI
+
+
+set ES_CLASSPATH=$CLASSPATH;"%ES_HOME%/lib/*"
+set ES_PARAMS=-Delasticsearch -Des.path.home="%ES_HOME%"
+
+"%JAVA_HOME%\bin\java" %JAVA_OPTS% %ES_JAVA_OPTS% %ES_PARAMS% -cp "%ES_CLASSPATH%" "org.elasticsearch.plugins.PluginManager" %*
+goto finally
+
+
+:err
+echo JAVA_HOME environment variable must be set!
+pause
+
+
+:finally
+
+ENDLOCAL

build.gradle

@@ -78,7 +78,7 @@ task explodedDist(dependsOn: [configurations.distLib], description: 'Builds a mi
     ant.delete { fileset(dir: explodedDistLibDir, includes: "slf4j-*.jar") } // no need for slf4j
     ant.delete { fileset(dir: explodedDistLibDir, includes: "jackson-*.jar") } // no need jackson, we jarjar it
     ant.delete { fileset(dir: explodedDistLibDir, includes: "joda-*.jar") } // no need joda, we jarjar it
-    ant.delete { fileset(dir: explodedDistLibDir, includes: "snakeyaml-*.jar") } // no need joda, we jarjar it
+    ant.delete { fileset(dir: explodedDistLibDir, includes: "snakeyaml-*.jar") } // no need snakeyaml, we jarjar it
 
     ant.chmod(dir: "$explodedDistDir/bin", perm: "ugo+rx", includes: "**/*")
 }
@@ -88,6 +88,7 @@ task zip(type: Zip, dependsOn: ['explodedDist']) {
     from(explodedDistDir) {
         into zipRootFolder
         exclude 'bin/elasticsearch'
+        exclude 'bin/plugin'
         exclude 'bin/service/elasticsearch'
         exclude 'bin/service/elasticsearch32'
         exclude 'bin/service/elasticsearch64'
@@ -96,6 +97,7 @@ task zip(type: Zip, dependsOn: ['explodedDist']) {
     from(explodedDistDir) {
         into zipRootFolder
         include 'bin/elasticsearch'
+        include 'bin/plugin'
         include 'bin/service/elasticsearch'
         include 'bin/service/elasticsearch32'
         include 'bin/service/elasticsearch64'

modules/elasticsearch/src/main/java/org/elasticsearch/plugins/PluginManager.java

@@ -0,0 +1,69 @@
+package org.elasticsearch.plugins;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.node.internal.InternalSettingsPerparer;
+import org.elasticsearch.util.Tuple;
+import org.elasticsearch.util.http.HttpDownloadHelper;
+import org.elasticsearch.util.settings.Settings;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URL;
+
+import static org.elasticsearch.util.settings.ImmutableSettings.Builder.*;
+
+/**
+ * @author kimchy (shay.banon)
+ */
+public class PluginManager {
+
+    private final Environment environment;
+
+    private final String url;
+
+    public PluginManager(Environment environment, String url) {
+        this.environment = environment;
+        this.url = url;
+    }
+
+    public void downloadPlugin(String name) throws IOException {
+        HttpDownloadHelper downloadHelper = new HttpDownloadHelper();
+
+        URL pluginUrl = new URL(url + "/" + name + "/elasticsearch-" + name + "-" + Version.number() + ".zip");
+        downloadHelper.download(pluginUrl, new File(environment.pluginsFile(), name + ".zip"), new HttpDownloadHelper.VerboseProgress(System.out));
+    }
+
+    public static void main(String[] args) {
+        Tuple<Settings, Environment> initialSettings = InternalSettingsPerparer.prepareSettings(EMPTY_SETTINGS, true);
+
+        if (!initialSettings.v2().pluginsFile().exists()) {
+            initialSettings.v2().pluginsFile().mkdirs();
+        }
+
+        PluginManager pluginManager = new PluginManager(initialSettings.v2(), "http://elasticsearch.googlecode.com/svn/plugins");
+
+        if (args.length < 1) {
+            System.out.println("Usage:");
+            System.out.println("    - get [list of plugin names]: Downloads all the listed plugins");
+        }
+        String command = args[0];
+        if (command.equals("get") || command.equals("-get") || command.equals("-g") || command.equals("--get")) {
+            if (args.length < 2) {
+                System.out.println("'get' requires an additional parameter with the plugin name");
+            }
+            for (int i = 1; i < args.length; i++) {
+                String pluginName = args[i];
+                System.out.print("-> Downloading " + pluginName + " ");
+                try {
+                    pluginManager.downloadPlugin(pluginName);
+                    System.out.println(" DONE");
+                } catch (IOException e) {
+                    System.out.println("Failed to download " + pluginName + ", reason: " + e.getMessage());
+                }
+            }
+        } else {
+            System.out.println("No command matching '" + command + "' found");
+        }
+    }
+}

modules/elasticsearch/src/main/java/org/elasticsearch/util/http/HttpDownloadHelper.java

@@ -0,0 +1,389 @@
+/*
+ * Licensed to Elastic Search and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Elastic Search licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.util.http;
+
+import javax.annotation.Nullable;
+import java.io.*;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.net.URLConnection;
+
+/**
+ * @author kimchy (shay.banon)
+ */
+public class HttpDownloadHelper {
+
+    private boolean useTimestamp = false;
+    private boolean skipExisting = false;
+    private long maxTime = 0;
+
+    public boolean download(URL source, File dest, @Nullable DownloadProgress progress) throws IOException {
+        if (dest.exists() && skipExisting) {
+            return true;
+        }
+
+        //don't do any progress, unless asked
+        if (progress == null) {
+            progress = new NullProgress();
+        }
+
+        //set the timestamp to the file date.
+        long timestamp = 0;
+
+        boolean hasTimestamp = false;
+        if (useTimestamp && dest.exists()) {
+            timestamp = dest.lastModified();
+            hasTimestamp = true;
+        }
+
+        GetThread getThread = new GetThread(source, dest, hasTimestamp, timestamp, progress);
+        getThread.setDaemon(true);
+        getThread.start();
+        try {
+            getThread.join(maxTime * 1000);
+        } catch (InterruptedException ie) {
+            // ignore
+        }
+
+        if (getThread.isAlive()) {
+            String msg = "The GET operation took longer than " + maxTime
+                    + " seconds, stopping it.";
+            getThread.closeStreams();
+            throw new IOException(msg);
+        }
+
+        return getThread.wasSuccessful();
+    }
+
+
+    /**
+     * Interface implemented for reporting
+     * progress of downloading.
+     */
+    public interface DownloadProgress {
+        /**
+         * begin a download
+         */
+        void beginDownload();
+
+        /**
+         * tick handler
+         */
+        void onTick();
+
+        /**
+         * end a download
+         */
+        void endDownload();
+    }
+
+    /**
+     * do nothing with progress info
+     */
+    public static class NullProgress implements DownloadProgress {
+
+        /**
+         * begin a download
+         */
+        public void beginDownload() {
+
+        }
+
+        /**
+         * tick handler
+         */
+        public void onTick() {
+        }
+
+        /**
+         * end a download
+         */
+        public void endDownload() {
+
+        }
+    }
+
+    /**
+     * verbose progress system prints to some output stream
+     */
+    public static class VerboseProgress implements DownloadProgress {
+        private int dots = 0;
+        // CheckStyle:VisibilityModifier OFF - bc
+        PrintStream out;
+        // CheckStyle:VisibilityModifier ON
+
+        /**
+         * Construct a verbose progress reporter.
+         *
+         * @param out the output stream.
+         */
+        public VerboseProgress(PrintStream out) {
+            this.out = out;
+        }
+
+        /**
+         * begin a download
+         */
+        public void beginDownload() {
+            dots = 0;
+        }
+
+        /**
+         * tick handler
+         */
+        public void onTick() {
+            out.print(".");
+            if (dots++ > 50) {
+                out.flush();
+                dots = 0;
+            }
+        }
+
+        /**
+         * end a download
+         */
+        public void endDownload() {
+            out.flush();
+        }
+    }
+
+    private class GetThread extends Thread {
+
+        private final URL source;
+        private final File dest;
+        private final boolean hasTimestamp;
+        private final long timestamp;
+        private final DownloadProgress progress;
+
+        private boolean success = false;
+        private IOException ioexception = null;
+        private InputStream is = null;
+        private OutputStream os = null;
+        private URLConnection connection;
+        private int redirections = 0;
+
+        GetThread(URL source, File dest, boolean h, long t, DownloadProgress p) {
+            this.source = source;
+            this.dest = dest;
+            hasTimestamp = h;
+            timestamp = t;
+            progress = p;
+        }
+
+        public void run() {
+            try {
+                success = get();
+            } catch (IOException ioex) {
+                ioexception = ioex;
+            }
+        }
+
+        private boolean get() throws IOException {
+
+            connection = openConnection(source);
+
+            if (connection == null) {
+                return false;
+            }
+
+            boolean downloadSucceeded = downloadFile();
+
+            //if (and only if) the use file time option is set, then
+            //the saved file now has its timestamp set to that of the
+            //downloaded file
+            if (downloadSucceeded && useTimestamp) {
+                updateTimeStamp();
+            }
+
+            return downloadSucceeded;
+        }
+
+
+        private boolean redirectionAllowed(URL aSource, URL aDest) throws IOException {
+            if (!(aSource.getProtocol().equals(aDest.getProtocol()) || ("http"
+                    .equals(aSource.getProtocol()) && "https".equals(aDest
+                    .getProtocol())))) {
+                String message = "Redirection detected from "
+                        + aSource.getProtocol() + " to " + aDest.getProtocol()
+                        + ". Protocol switch unsafe, not allowed.";
+                throw new IOException(message);
+            }
+
+            redirections++;
+            if (redirections > 5) {
+                String message = "More than " + 5 + " times redirected, giving up";
+                throw new IOException(message);
+            }
+
+
+            return true;
+        }
+
+        private URLConnection openConnection(URL aSource) throws IOException {
+
+            // set up the URL connection
+            URLConnection connection = aSource.openConnection();
+            // modify the headers
+            // NB: things like user authentication could go in here too.
+            if (hasTimestamp) {
+                connection.setIfModifiedSince(timestamp);
+            }
+
+            if (connection instanceof HttpURLConnection) {
+                ((HttpURLConnection) connection).setInstanceFollowRedirects(false);
+                ((HttpURLConnection) connection).setUseCaches(true);
+            }
+            // connect to the remote site (may take some time)
+            connection.connect();
+
+            // First check on a 301 / 302 (moved) response (HTTP only)
+            if (connection instanceof HttpURLConnection) {
+                HttpURLConnection httpConnection = (HttpURLConnection) connection;
+                int responseCode = httpConnection.getResponseCode();
+                if (responseCode == HttpURLConnection.HTTP_MOVED_PERM ||
+                        responseCode == HttpURLConnection.HTTP_MOVED_TEMP ||
+                        responseCode == HttpURLConnection.HTTP_SEE_OTHER) {
+                    String newLocation = httpConnection.getHeaderField("Location");
+                    String message = aSource
+                            + (responseCode == HttpURLConnection.HTTP_MOVED_PERM ? " permanently"
+                            : "") + " moved to " + newLocation;
+                    URL newURL = new URL(newLocation);
+                    if (!redirectionAllowed(aSource, newURL)) {
+                        return null;
+                    }
+                    return openConnection(newURL);
+                }
+                // next test for a 304 result (HTTP only)
+                long lastModified = httpConnection.getLastModified();
+                if (responseCode == HttpURLConnection.HTTP_NOT_MODIFIED
+                        || (lastModified != 0 && hasTimestamp && timestamp >= lastModified)) {
+                    // not modified so no file download. just return
+                    // instead and trace out something so the user
+                    // doesn't think that the download happened when it
+                    // didn't
+                    return null;
+                }
+                // test for 401 result (HTTP only)
+                if (responseCode == HttpURLConnection.HTTP_UNAUTHORIZED) {
+                    String message = "HTTP Authorization failure";
+                    throw new IOException(message);
+                }
+            }
+
+            //REVISIT: at this point even non HTTP connections may
+            //support the if-modified-since behaviour -we just check
+            //the date of the content and skip the write if it is not
+            //newer. Some protocols (FTP) don't include dates, of
+            //course.
+            return connection;
+        }
+
+        private boolean downloadFile() throws FileNotFoundException, IOException {
+            IOException lastEx = null;
+            for (int i = 0; i < 3; i++) {
+                // this three attempt trick is to get round quirks in different
+                // Java implementations. Some of them take a few goes to bind
+                // property; we ignore the first couple of such failures.
+                try {
+                    is = connection.getInputStream();
+                    break;
+                } catch (IOException ex) {
+                    lastEx = ex;
+                }
+            }
+            if (is == null) {
+                throw new IOException("Can't get " + source + " to " + dest, lastEx);
+            }
+
+            os = new FileOutputStream(dest);
+            progress.beginDownload();
+            boolean finished = false;
+            try {
+                byte[] buffer = new byte[1024 * 100];
+                int length;
+                while (!isInterrupted() && (length = is.read(buffer)) >= 0) {
+                    os.write(buffer, 0, length);
+                    progress.onTick();
+                }
+                finished = !isInterrupted();
+            } finally {
+                try {
+                    os.close();
+                } catch (IOException e) {
+                    // ignore
+                }
+                try {
+                    is.close();
+                } catch (IOException e) {
+                    // ignore
+                }
+
+                // we have started to (over)write dest, but failed.
+                // Try to delete the garbage we'd otherwise leave
+                // behind.
+                if (!finished) {
+                    dest.delete();
+                }
+            }
+            progress.endDownload();
+            return true;
+        }
+
+        private void updateTimeStamp() {
+            long remoteTimestamp = connection.getLastModified();
+            if (remoteTimestamp != 0) {
+                dest.setLastModified(remoteTimestamp);
+            }
+        }
+
+        /**
+         * Has the download completed successfully?
+         *
+         * <p>Re-throws any exception caught during executaion.</p>
+         */
+        boolean wasSuccessful() throws IOException {
+            if (ioexception != null) {
+                throw ioexception;
+            }
+            return success;
+        }
+
+        /**
+         * Closes streams, interrupts the download, may delete the
+         * output file.
+         */
+        void closeStreams() {
+            interrupt();
+            try {
+                os.close();
+            } catch (IOException e) {
+                // ignore
+            }
+            try {
+                is.close();
+            } catch (IOException e) {
+                // ignore
+            }
+            if (!success && dest.exists()) {
+                dest.delete();
+            }
+        }
+    }
+}