From b0e1c58474dd5de39a2b26cca00ac85f72d8b55d Mon Sep 17 00:00:00 2001 From: kimchy Date: Mon, 3 May 2010 10:49:50 +0300 Subject: [PATCH] Add online plugin repository and a 'plugin' command to download them, closes #157 --- bin/plugin | 33 ++ bin/plugin.bat | 25 ++ build.gradle | 4 +- .../elasticsearch/plugins/PluginManager.java | 69 ++++ .../util/http/HttpDownloadHelper.java | 389 ++++++++++++++++++ 5 files changed, 519 insertions(+), 1 deletion(-) create mode 100644 bin/plugin create mode 100644 bin/plugin.bat create mode 100644 modules/elasticsearch/src/main/java/org/elasticsearch/plugins/PluginManager.java create mode 100644 modules/elasticsearch/src/main/java/org/elasticsearch/util/http/HttpDownloadHelper.java diff --git a/bin/plugin b/bin/plugin new file mode 100644 index 00000000000..9964da5c870 --- /dev/null +++ b/bin/plugin @@ -0,0 +1,33 @@ +#!/bin/sh + +SCRIPT="$0" + +# SCRIPT may be an arbitrarily deep series of symlinks. Loop until we have the concrete path. +while [ -h "$SCRIPT" ] ; do + ls=`ls -ld "$SCRIPT"` + # Drop everything prior to -> + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + SCRIPT="$link" + else + SCRIPT=`dirname "$SCRIPT"`/"$link" + fi +done + +# determine elasticsearch home +ES_HOME=`dirname "$SCRIPT"`/.. + +# make ELASTICSEARCH_HOME absolute +ES_HOME=`cd $ES_HOME; pwd` + + +if [ -x $JAVA_HOME/bin/java ]; then + JAVA=$JAVA_HOME/bin/java +else + JAVA=`which java` +fi + +CLASSPATH=$CLASSPATH:$ES_HOME/lib/* + +$JAVA -Delasticsearch -Des.path.home=$ES_HOME -cp $CLASSPATH org.elasticsearch.plugins.PluginManager $* + \ No newline at end of file diff --git a/bin/plugin.bat b/bin/plugin.bat new file mode 100644 index 00000000000..24868fd7072 --- /dev/null +++ b/bin/plugin.bat @@ -0,0 +1,25 @@ +@echo off + +SETLOCAL + +if NOT DEFINED JAVA_HOME goto err + +set SCRIPT_DIR=%~dp0 +for %%I in ("%SCRIPT_DIR%..") do set ES_HOME=%%~dpfI + + +set ES_CLASSPATH=$CLASSPATH;"%ES_HOME%/lib/*" +set ES_PARAMS=-Delasticsearch -Des.path.home="%ES_HOME%" + +"%JAVA_HOME%\bin\java" %JAVA_OPTS% %ES_JAVA_OPTS% %ES_PARAMS% -cp "%ES_CLASSPATH%" "org.elasticsearch.plugins.PluginManager" %* +goto finally + + +:err +echo JAVA_HOME environment variable must be set! +pause + + +:finally + +ENDLOCAL \ No newline at end of file diff --git a/build.gradle b/build.gradle index b5a445f7730..8bc13b3261a 100644 --- a/build.gradle +++ b/build.gradle @@ -78,7 +78,7 @@ task explodedDist(dependsOn: [configurations.distLib], description: 'Builds a mi ant.delete { fileset(dir: explodedDistLibDir, includes: "slf4j-*.jar") } // no need for slf4j ant.delete { fileset(dir: explodedDistLibDir, includes: "jackson-*.jar") } // no need jackson, we jarjar it ant.delete { fileset(dir: explodedDistLibDir, includes: "joda-*.jar") } // no need joda, we jarjar it - ant.delete { fileset(dir: explodedDistLibDir, includes: "snakeyaml-*.jar") } // no need joda, we jarjar it + ant.delete { fileset(dir: explodedDistLibDir, includes: "snakeyaml-*.jar") } // no need snakeyaml, we jarjar it ant.chmod(dir: "$explodedDistDir/bin", perm: "ugo+rx", includes: "**/*") } @@ -88,6 +88,7 @@ task zip(type: Zip, dependsOn: ['explodedDist']) { from(explodedDistDir) { into zipRootFolder exclude 'bin/elasticsearch' + exclude 'bin/plugin' exclude 'bin/service/elasticsearch' exclude 'bin/service/elasticsearch32' exclude 'bin/service/elasticsearch64' @@ -96,6 +97,7 @@ task zip(type: Zip, dependsOn: ['explodedDist']) { from(explodedDistDir) { into zipRootFolder include 'bin/elasticsearch' + include 'bin/plugin' include 'bin/service/elasticsearch' include 'bin/service/elasticsearch32' include 'bin/service/elasticsearch64' diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/plugins/PluginManager.java b/modules/elasticsearch/src/main/java/org/elasticsearch/plugins/PluginManager.java new file mode 100644 index 00000000000..9f6d2266edc --- /dev/null +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/plugins/PluginManager.java @@ -0,0 +1,69 @@ +package org.elasticsearch.plugins; + +import org.elasticsearch.Version; +import org.elasticsearch.env.Environment; +import org.elasticsearch.node.internal.InternalSettingsPerparer; +import org.elasticsearch.util.Tuple; +import org.elasticsearch.util.http.HttpDownloadHelper; +import org.elasticsearch.util.settings.Settings; + +import java.io.File; +import java.io.IOException; +import java.net.URL; + +import static org.elasticsearch.util.settings.ImmutableSettings.Builder.*; + +/** + * @author kimchy (shay.banon) + */ +public class PluginManager { + + private final Environment environment; + + private final String url; + + public PluginManager(Environment environment, String url) { + this.environment = environment; + this.url = url; + } + + public void downloadPlugin(String name) throws IOException { + HttpDownloadHelper downloadHelper = new HttpDownloadHelper(); + + URL pluginUrl = new URL(url + "/" + name + "/elasticsearch-" + name + "-" + Version.number() + ".zip"); + downloadHelper.download(pluginUrl, new File(environment.pluginsFile(), name + ".zip"), new HttpDownloadHelper.VerboseProgress(System.out)); + } + + public static void main(String[] args) { + Tuple initialSettings = InternalSettingsPerparer.prepareSettings(EMPTY_SETTINGS, true); + + if (!initialSettings.v2().pluginsFile().exists()) { + initialSettings.v2().pluginsFile().mkdirs(); + } + + PluginManager pluginManager = new PluginManager(initialSettings.v2(), "http://elasticsearch.googlecode.com/svn/plugins"); + + if (args.length < 1) { + System.out.println("Usage:"); + System.out.println(" - get [list of plugin names]: Downloads all the listed plugins"); + } + String command = args[0]; + if (command.equals("get") || command.equals("-get") || command.equals("-g") || command.equals("--get")) { + if (args.length < 2) { + System.out.println("'get' requires an additional parameter with the plugin name"); + } + for (int i = 1; i < args.length; i++) { + String pluginName = args[i]; + System.out.print("-> Downloading " + pluginName + " "); + try { + pluginManager.downloadPlugin(pluginName); + System.out.println(" DONE"); + } catch (IOException e) { + System.out.println("Failed to download " + pluginName + ", reason: " + e.getMessage()); + } + } + } else { + System.out.println("No command matching '" + command + "' found"); + } + } +} diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/util/http/HttpDownloadHelper.java b/modules/elasticsearch/src/main/java/org/elasticsearch/util/http/HttpDownloadHelper.java new file mode 100644 index 00000000000..69033bf3455 --- /dev/null +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/util/http/HttpDownloadHelper.java @@ -0,0 +1,389 @@ +/* + * Licensed to Elastic Search and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Elastic Search licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.util.http; + +import javax.annotation.Nullable; +import java.io.*; +import java.net.HttpURLConnection; +import java.net.URL; +import java.net.URLConnection; + +/** + * @author kimchy (shay.banon) + */ +public class HttpDownloadHelper { + + private boolean useTimestamp = false; + private boolean skipExisting = false; + private long maxTime = 0; + + public boolean download(URL source, File dest, @Nullable DownloadProgress progress) throws IOException { + if (dest.exists() && skipExisting) { + return true; + } + + //don't do any progress, unless asked + if (progress == null) { + progress = new NullProgress(); + } + + //set the timestamp to the file date. + long timestamp = 0; + + boolean hasTimestamp = false; + if (useTimestamp && dest.exists()) { + timestamp = dest.lastModified(); + hasTimestamp = true; + } + + GetThread getThread = new GetThread(source, dest, hasTimestamp, timestamp, progress); + getThread.setDaemon(true); + getThread.start(); + try { + getThread.join(maxTime * 1000); + } catch (InterruptedException ie) { + // ignore + } + + if (getThread.isAlive()) { + String msg = "The GET operation took longer than " + maxTime + + " seconds, stopping it."; + getThread.closeStreams(); + throw new IOException(msg); + } + + return getThread.wasSuccessful(); + } + + + /** + * Interface implemented for reporting + * progress of downloading. + */ + public interface DownloadProgress { + /** + * begin a download + */ + void beginDownload(); + + /** + * tick handler + */ + void onTick(); + + /** + * end a download + */ + void endDownload(); + } + + /** + * do nothing with progress info + */ + public static class NullProgress implements DownloadProgress { + + /** + * begin a download + */ + public void beginDownload() { + + } + + /** + * tick handler + */ + public void onTick() { + } + + /** + * end a download + */ + public void endDownload() { + + } + } + + /** + * verbose progress system prints to some output stream + */ + public static class VerboseProgress implements DownloadProgress { + private int dots = 0; + // CheckStyle:VisibilityModifier OFF - bc + PrintStream out; + // CheckStyle:VisibilityModifier ON + + /** + * Construct a verbose progress reporter. + * + * @param out the output stream. + */ + public VerboseProgress(PrintStream out) { + this.out = out; + } + + /** + * begin a download + */ + public void beginDownload() { + dots = 0; + } + + /** + * tick handler + */ + public void onTick() { + out.print("."); + if (dots++ > 50) { + out.flush(); + dots = 0; + } + } + + /** + * end a download + */ + public void endDownload() { + out.flush(); + } + } + + private class GetThread extends Thread { + + private final URL source; + private final File dest; + private final boolean hasTimestamp; + private final long timestamp; + private final DownloadProgress progress; + + private boolean success = false; + private IOException ioexception = null; + private InputStream is = null; + private OutputStream os = null; + private URLConnection connection; + private int redirections = 0; + + GetThread(URL source, File dest, boolean h, long t, DownloadProgress p) { + this.source = source; + this.dest = dest; + hasTimestamp = h; + timestamp = t; + progress = p; + } + + public void run() { + try { + success = get(); + } catch (IOException ioex) { + ioexception = ioex; + } + } + + private boolean get() throws IOException { + + connection = openConnection(source); + + if (connection == null) { + return false; + } + + boolean downloadSucceeded = downloadFile(); + + //if (and only if) the use file time option is set, then + //the saved file now has its timestamp set to that of the + //downloaded file + if (downloadSucceeded && useTimestamp) { + updateTimeStamp(); + } + + return downloadSucceeded; + } + + + private boolean redirectionAllowed(URL aSource, URL aDest) throws IOException { + if (!(aSource.getProtocol().equals(aDest.getProtocol()) || ("http" + .equals(aSource.getProtocol()) && "https".equals(aDest + .getProtocol())))) { + String message = "Redirection detected from " + + aSource.getProtocol() + " to " + aDest.getProtocol() + + ". Protocol switch unsafe, not allowed."; + throw new IOException(message); + } + + redirections++; + if (redirections > 5) { + String message = "More than " + 5 + " times redirected, giving up"; + throw new IOException(message); + } + + + return true; + } + + private URLConnection openConnection(URL aSource) throws IOException { + + // set up the URL connection + URLConnection connection = aSource.openConnection(); + // modify the headers + // NB: things like user authentication could go in here too. + if (hasTimestamp) { + connection.setIfModifiedSince(timestamp); + } + + if (connection instanceof HttpURLConnection) { + ((HttpURLConnection) connection).setInstanceFollowRedirects(false); + ((HttpURLConnection) connection).setUseCaches(true); + } + // connect to the remote site (may take some time) + connection.connect(); + + // First check on a 301 / 302 (moved) response (HTTP only) + if (connection instanceof HttpURLConnection) { + HttpURLConnection httpConnection = (HttpURLConnection) connection; + int responseCode = httpConnection.getResponseCode(); + if (responseCode == HttpURLConnection.HTTP_MOVED_PERM || + responseCode == HttpURLConnection.HTTP_MOVED_TEMP || + responseCode == HttpURLConnection.HTTP_SEE_OTHER) { + String newLocation = httpConnection.getHeaderField("Location"); + String message = aSource + + (responseCode == HttpURLConnection.HTTP_MOVED_PERM ? " permanently" + : "") + " moved to " + newLocation; + URL newURL = new URL(newLocation); + if (!redirectionAllowed(aSource, newURL)) { + return null; + } + return openConnection(newURL); + } + // next test for a 304 result (HTTP only) + long lastModified = httpConnection.getLastModified(); + if (responseCode == HttpURLConnection.HTTP_NOT_MODIFIED + || (lastModified != 0 && hasTimestamp && timestamp >= lastModified)) { + // not modified so no file download. just return + // instead and trace out something so the user + // doesn't think that the download happened when it + // didn't + return null; + } + // test for 401 result (HTTP only) + if (responseCode == HttpURLConnection.HTTP_UNAUTHORIZED) { + String message = "HTTP Authorization failure"; + throw new IOException(message); + } + } + + //REVISIT: at this point even non HTTP connections may + //support the if-modified-since behaviour -we just check + //the date of the content and skip the write if it is not + //newer. Some protocols (FTP) don't include dates, of + //course. + return connection; + } + + private boolean downloadFile() throws FileNotFoundException, IOException { + IOException lastEx = null; + for (int i = 0; i < 3; i++) { + // this three attempt trick is to get round quirks in different + // Java implementations. Some of them take a few goes to bind + // property; we ignore the first couple of such failures. + try { + is = connection.getInputStream(); + break; + } catch (IOException ex) { + lastEx = ex; + } + } + if (is == null) { + throw new IOException("Can't get " + source + " to " + dest, lastEx); + } + + os = new FileOutputStream(dest); + progress.beginDownload(); + boolean finished = false; + try { + byte[] buffer = new byte[1024 * 100]; + int length; + while (!isInterrupted() && (length = is.read(buffer)) >= 0) { + os.write(buffer, 0, length); + progress.onTick(); + } + finished = !isInterrupted(); + } finally { + try { + os.close(); + } catch (IOException e) { + // ignore + } + try { + is.close(); + } catch (IOException e) { + // ignore + } + + // we have started to (over)write dest, but failed. + // Try to delete the garbage we'd otherwise leave + // behind. + if (!finished) { + dest.delete(); + } + } + progress.endDownload(); + return true; + } + + private void updateTimeStamp() { + long remoteTimestamp = connection.getLastModified(); + if (remoteTimestamp != 0) { + dest.setLastModified(remoteTimestamp); + } + } + + /** + * Has the download completed successfully? + * + *

Re-throws any exception caught during executaion.

+ */ + boolean wasSuccessful() throws IOException { + if (ioexception != null) { + throw ioexception; + } + return success; + } + + /** + * Closes streams, interrupts the download, may delete the + * output file. + */ + void closeStreams() { + interrupt(); + try { + os.close(); + } catch (IOException e) { + // ignore + } + try { + is.close(); + } catch (IOException e) { + // ignore + } + if (!success && dest.exists()) { + dest.delete(); + } + } + } +}