honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Plugins: Allow plugins to serve a _site, automatically download github plugins, closes #978.

This commit is contained in:
kimchy 2011-05-28 18:43:29 +03:00
parent 4cd2f79972
commit 3652d57667
6 changed files with 287 additions and 115 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
modules/elasticsearch/src/main/java/org/elasticsearch/common/http/client/HttpDownloadHelper.java
View File

@ -21,7 +21,13 @@ package org.elasticsearch.common.http.client;
import org.elasticsearch.common.Nullable;
import java.io.*;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.PrintStream;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLConnection;
@ -143,6 +149,7 @@ public class HttpDownloadHelper {
* begin a download
*/
public void beginDownload() {
out.print("Downloading ");
dots = 0;
}
@ -161,6 +168,7 @@ public class HttpDownloadHelper {
* end a download
*/
public void endDownload() {
out.println("DONE");
out.flush();
}
}

33
modules/elasticsearch/src/main/java/org/elasticsearch/common/io/FileSystemUtils.java
View File

@ -19,7 +19,12 @@
package org.elasticsearch.common.io;
import java.io.*;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InterruptedIOException;
import java.io.RandomAccessFile;
import java.nio.channels.FileChannel;
import java.util.ArrayList;
import java.util.List;
@ -57,6 +62,32 @@ public class FileSystemUtils {
return files.size();
}
public static boolean hasExtensions(File root, String... extensions) {
if (root != null && root.exists()) {
if (root.isDirectory()) {
File[] children = root.listFiles();
if (children != null) {
for (File child : children) {
if (child.isDirectory()) {
boolean has = hasExtensions(child, extensions);
if (has) {
return true;
}
} else {
for (String extension : extensions) {
if (child.getName().endsWith(extension)) {
return true;
}
}
}
}
}
}
}
return false;
}
public static boolean deleteRecursively(File root) {
return deleteRecursively(root, true);
}

162
modules/elasticsearch/src/main/java/org/elasticsearch/http/HttpServer.java
View File

@ -23,15 +23,15 @@ import org.elasticsearch.ElasticSearchException;
import org.elasticsearch.action.admin.cluster.node.info.TransportNodesInfoAction;
import org.elasticsearch.common.component.AbstractLifecycleComponent;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.path.PathTrie;
import org.elasticsearch.common.io.Streams;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.env.Environment;
import org.elasticsearch.rest.BytesRestResponse;
import org.elasticsearch.rest.RestController;
import org.elasticsearch.rest.RestRequest;
import org.elasticsearch.rest.StringRestResponse;
import org.elasticsearch.rest.XContentThrowableRestResponse;
import org.elasticsearch.rest.support.RestUtils;
import org.elasticsearch.threadpool.ThreadPool;
import java.io.File;
import java.io.IOException;
import static org.elasticsearch.rest.RestStatus.*;
@ -41,49 +41,35 @@ import static org.elasticsearch.rest.RestStatus.*;
*/
public class HttpServer extends AbstractLifecycleComponent<HttpServer> {
private final HttpServerTransport transport;
private final Environment environment;
private final ThreadPool threadPool;
private final HttpServerTransport transport;
private final RestController restController;
private final TransportNodesInfoAction nodesInfoAction;
private final PathTrie<HttpServerHandler> getHandlers = new PathTrie<HttpServerHandler>(RestUtils.REST_DECODER);
private final PathTrie<HttpServerHandler> postHandlers = new PathTrie<HttpServerHandler>(RestUtils.REST_DECODER);
private final PathTrie<HttpServerHandler> putHandlers = new PathTrie<HttpServerHandler>(RestUtils.REST_DECODER);
private final PathTrie<HttpServerHandler> deleteHandlers = new PathTrie<HttpServerHandler>(RestUtils.REST_DECODER);
private final PathTrie<HttpServerHandler> headHandlers = new PathTrie<HttpServerHandler>(RestUtils.REST_DECODER);
private final PathTrie<HttpServerHandler> optionsHandlers = new PathTrie<HttpServerHandler>(RestUtils.REST_DECODER);
@Inject public HttpServer(Settings settings, HttpServerTransport transport, ThreadPool threadPool,
@Inject public HttpServer(Settings settings, Environment environment, HttpServerTransport transport,
RestController restController, TransportNodesInfoAction nodesInfoAction) {
super(settings);
this.environment = environment;
this.transport = transport;
this.threadPool = threadPool;
this.restController = restController;
this.nodesInfoAction = nodesInfoAction;
transport.httpServerAdapter(new HttpServerAdapter() {
@Override public void dispatchRequest(HttpRequest request, HttpChannel channel) {
internalDispatchRequest(request, channel);
}
});
transport.httpServerAdapter(new Dispatcher(this));
}
public void registerHandler(HttpRequest.Method method, String path, HttpServerHandler handler) {
if (method == HttpRequest.Method.GET) {
getHandlers.insert(path, handler);
} else if (method == HttpRequest.Method.POST) {
postHandlers.insert(path, handler);
} else if (method == HttpRequest.Method.PUT) {
putHandlers.insert(path, handler);
} else if (method == HttpRequest.Method.DELETE) {
deleteHandlers.insert(path, handler);
} else if (method == RestRequest.Method.HEAD) {
headHandlers.insert(path, handler);
} else if (method == RestRequest.Method.OPTIONS) {
optionsHandlers.insert(path, handler);
static class Dispatcher implements HttpServerAdapter {
private final HttpServer server;
Dispatcher(HttpServer server) {
this.server = server;
}
@Override public void dispatchRequest(HttpRequest request, HttpChannel channel) {
server.internalDispatchRequest(request, channel);
}
}
@ -104,72 +90,62 @@ public class HttpServer extends AbstractLifecycleComponent<HttpServer> {
transport.close();
}
void internalDispatchRequest(final HttpRequest request, final HttpChannel channel) {
final HttpServerHandler httpHandler = getHandler(request);
if (httpHandler == null) {
// if nothing was dispatched by the rest request, send either error or default handling per method
if (!restController.dispatchRequest(request, channel)) {
if (request.method() == RestRequest.Method.OPTIONS) {
// when we have OPTIONS request, simply send OK by default (with the Access Control Origin header which gets automatically added)
StringRestResponse response = new StringRestResponse(OK);
channel.sendResponse(response);
} else {
channel.sendResponse(new StringRestResponse(BAD_REQUEST, "No handler found for uri [" + request.uri() + "] and method [" + request.method() + "]"));
}
}
} else {
if (httpHandler.spawn()) {
threadPool.cached().execute(new Runnable() {
@Override public void run() {
try {
httpHandler.handleRequest(request, channel);
} catch (Exception e) {
try {
channel.sendResponse(new XContentThrowableRestResponse(request, e));
} catch (IOException e1) {
logger.error("Failed to send failure response for uri [" + request.uri() + "]", e1);
}
}
}
});
public void internalDispatchRequest(final HttpRequest request, final HttpChannel channel) {
if (request.rawPath().startsWith("/_plugin/")) {
handlePluginSite(request, channel);
return;
}
if (!restController.dispatchRequest(request, channel)) {
if (request.method() == RestRequest.Method.OPTIONS) {
// when we have OPTIONS request, simply send OK by default (with the Access Control Origin header which gets automatically added)
StringRestResponse response = new StringRestResponse(OK);
channel.sendResponse(response);
} else {
try {
httpHandler.handleRequest(request, channel);
} catch (Exception e) {
try {
channel.sendResponse(new XContentThrowableRestResponse(request, e));
} catch (IOException e1) {
logger.error("Failed to send failure response for uri [" + request.uri() + "]", e1);
}
}
channel.sendResponse(new StringRestResponse(BAD_REQUEST, "No handler found for uri [" + request.uri() + "] and method [" + request.method() + "]"));
}
}
}
private HttpServerHandler getHandler(HttpRequest request) {
String path = getPath(request);
HttpRequest.Method method = request.method();
if (method == HttpRequest.Method.GET) {
return getHandlers.retrieve(path, request.params());
} else if (method == HttpRequest.Method.POST) {
return postHandlers.retrieve(path, request.params());
} else if (method == HttpRequest.Method.PUT) {
return putHandlers.retrieve(path, request.params());
} else if (method == HttpRequest.Method.DELETE) {
return deleteHandlers.retrieve(path, request.params());
} else if (method == RestRequest.Method.HEAD) {
return headHandlers.retrieve(path, request.params());
} else if (method == RestRequest.Method.OPTIONS) {
return optionsHandlers.retrieve(path, request.params());
} else {
return null;
private void handlePluginSite(HttpRequest request, HttpChannel channel) {
if (request.method() != RestRequest.Method.GET) {
channel.sendResponse(new StringRestResponse(FORBIDDEN));
return;
}
}
int i1 = request.rawPath().indexOf('/', 9);
if (i1 == -1) {
channel.sendResponse(new StringRestResponse(NOT_FOUND));
return;
}
String pluginName = request.rawPath().substring(9, i1);
String sitePath = request.rawPath().substring(i1 + 1);
private String getPath(HttpRequest request) {
// we use rawPath since we don't want to decode it while processing the path resolution
// so we can handle things like:
// my_index/my_type/http%3A%2F%2Fwww.google.com
return request.rawPath();
if (sitePath.length() == 0) {
sitePath = "/index.html";
}
// Convert file separators.
sitePath = sitePath.replace('/', File.separatorChar);
// this is a plugin provided site, serve it as static files from the plugin location
File siteFile = new File(new File(environment.pluginsFile(), pluginName), "_site");
File file = new File(siteFile, sitePath);
if (!file.exists() || file.isHidden()) {
channel.sendResponse(new StringRestResponse(NOT_FOUND));
return;
}
if (!file.isFile()) {
channel.sendResponse(new StringRestResponse(FORBIDDEN));
return;
}
if (!file.getAbsolutePath().startsWith(siteFile.getAbsolutePath())) {
channel.sendResponse(new StringRestResponse(FORBIDDEN));
return;
}
try {
byte[] data = Streams.copyToByteArray(file);
channel.sendResponse(new BytesRestResponse(data, ""));
} catch (IOException e) {
channel.sendResponse(new StringRestResponse(INTERNAL_SERVER_ERROR));
}
}
}

129
modules/elasticsearch/src/main/java/org/elasticsearch/plugins/PluginManager.java
View File

@ -26,24 +26,112 @@ public class PluginManager {
private final Environment environment;
private final String url;
private String url;
public PluginManager(Environment environment, String url) {
this.environment = environment;
this.url = url;
}
public void downloadPlugin(String name) throws IOException {
public void downloadAndExtract(String name) throws IOException {
HttpDownloadHelper downloadHelper = new HttpDownloadHelper();
File pluginFile = new File(url + "/" + name + "/elasticsearch-" + name + "-" + Version.number() + ".zip");
boolean downloaded = false;
String filterZipName = null;
if (!pluginFile.exists()) {
pluginFile = new File(url + "/elasticsearch-" + name + "-" + Version.number() + ".zip");
if (!pluginFile.exists()) {
URL pluginUrl = new URL(url + "/" + name + "/elasticsearch-" + name + "-" + Version.number() + ".zip");
System.out.println("Downloading plugin from " + pluginUrl.toExternalForm());
pluginFile = new File(environment.pluginsFile(), name + ".zip");
downloadHelper.download(pluginUrl, pluginFile, new HttpDownloadHelper.VerboseProgress(System.out));
if (url != null) {
URL pluginUrl = new URL(url);
System.out.println("Trying " + pluginUrl.toExternalForm() + "...");
try {
downloadHelper.download(pluginUrl, pluginFile, new HttpDownloadHelper.VerboseProgress(System.out));
downloaded = true;
} catch (IOException e) {
// ignore
}
} else {
url = "http://elasticsearch.googlecode.com/svn/plugins";
}
if (!downloaded) {
if (name.indexOf('/') != -1) {
// github repo
String[] elements = name.split("/");
String userName = elements[0];
String repoName = elements[1];
String version = null;
if (elements.length > 2) {
version = elements[2];
}
filterZipName = userName + "-" + repoName;
// the installation file should not include the userName, just the repoName
name = repoName;
if (name.startsWith("elasticsearch-")) {
// remove elasticsearch- prefix
name = name.substring("elasticsearch-".length());
} else if (name.startsWith("es-")) {
// remove es- prefix
name = name.substring("es-".length());
}
pluginFile = new File(environment.pluginsFile(), name + ".zip");
if (version == null) {
// try with ES version from downloads
URL pluginUrl = new URL("http://github.com/downloads/" + userName + "/" + repoName + "/" + repoName + "-" + Version.number());
System.out.println("Trying " + pluginUrl.toExternalForm() + "...");
try {
downloadHelper.download(pluginUrl, pluginFile, new HttpDownloadHelper.VerboseProgress(System.out));
downloaded = true;
} catch (IOException e) {
// try a tag with ES version
pluginUrl = new URL("http://github.com/" + userName + "/" + repoName + "/zipball/v" + Version.number());
System.out.println("Trying " + pluginUrl.toExternalForm() + "...");
try {
downloadHelper.download(pluginUrl, pluginFile, new HttpDownloadHelper.VerboseProgress(System.out));
downloaded = true;
} catch (IOException e1) {
// download master
pluginUrl = new URL("http://github.com/" + userName + "/" + repoName + "/zipball/master");
System.out.println("Trying " + pluginUrl.toExternalForm() + "...");
try {
downloadHelper.download(pluginUrl, pluginFile, new HttpDownloadHelper.VerboseProgress(System.out));
downloaded = true;
} catch (IOException e2) {
// ignore
}
}
}
} else {
// download explicit version
URL pluginUrl = new URL("http://github.com/downloads/" + userName + "/" + repoName + "/" + repoName + "-" + version);
System.out.println("Trying " + pluginUrl.toExternalForm() + "...");
try {
downloadHelper.download(pluginUrl, pluginFile, new HttpDownloadHelper.VerboseProgress(System.out));
downloaded = true;
} catch (IOException e) {
// try a tag with ES version
pluginUrl = new URL("http://github.com/" + userName + "/" + repoName + "/zipball/v" + version);
System.out.println("Trying " + pluginUrl.toExternalForm() + "...");
try {
downloadHelper.download(pluginUrl, pluginFile, new HttpDownloadHelper.VerboseProgress(System.out));
downloaded = true;
} catch (IOException e1) {
// ignore
}
}
}
} else {
URL pluginUrl = new URL(url + "/" + name + "/elasticsearch-" + name + "-" + Version.number() + ".zip");
System.out.println("Trying " + pluginUrl.toExternalForm() + "...");
try {
downloadHelper.download(pluginUrl, pluginFile, new HttpDownloadHelper.VerboseProgress(System.out));
downloaded = true;
} catch (IOException e) {
// ignore
}
}
}
} else {
System.out.println("Using plugin from local fs: " + pluginFile.getAbsolutePath());
}
@ -51,6 +139,10 @@ public class PluginManager {
System.out.println("Using plugin from local fs: " + pluginFile.getAbsolutePath());
}
if (!downloaded) {
throw new IOException("failed to download");
}
// extract the plugin
File extractLocation = new File(environment.pluginsFile(), name);
ZipFile zipFile = null;
@ -59,10 +151,15 @@ public class PluginManager {
Enumeration<? extends ZipEntry> zipEntries = zipFile.entries();
while (zipEntries.hasMoreElements()) {
ZipEntry zipEntry = zipEntries.nextElement();
if (!(zipEntry.getName().endsWith(".jar") || zipEntry.getName().endsWith(".zip"))) {
if (zipEntry.isDirectory()) {
continue;
}
String zipName = zipEntry.getName().replace('\\', '/');
if (filterZipName != null) {
if (zipName.startsWith(filterZipName)) {
zipName = zipName.substring(zipName.indexOf('/'));
}
}
File target = new File(extractLocation, zipName);
target.getParentFile().mkdirs();
Streams.copy(zipFile.getInputStream(zipEntry), new FileOutputStream(target));
@ -79,6 +176,19 @@ public class PluginManager {
}
pluginFile.delete();
}
// try and identify the plugin type, see if it has no .class or .jar files in it
// so its probably a _site, and it it does not have a _site in it, move everything to _site
if (!new File(extractLocation, "_site").exists()) {
if (!FileSystemUtils.hasExtensions(extractLocation, ".class", ".jar")) {
System.out.println("Identified as a _site plugin, moving to _site structure ...");
File site = new File(extractLocation, "_site");
File tmpLocation = new File(environment.pluginsFile(), name + ".tmp");
extractLocation.renameTo(tmpLocation);
extractLocation.mkdirs();
tmpLocation.renameTo(site);
}
}
}
public void removePlugin(String name) throws IOException {
@ -99,7 +209,7 @@ public class PluginManager {
initialSettings.v2().pluginsFile().mkdirs();
}
String url = "http://elasticsearch.googlecode.com/svn/plugins";
String url = null;
for (int i = 0; i < args.length; i++) {
if ("url".equals(args[i]) || "-url".equals(args[i])) {
url = args[i + 1];
@ -119,10 +229,9 @@ public class PluginManager {
String command = args[c];
if (command.equals("install") || command.equals("-install")) {
String pluginName = args[++c];
System.out.print("-> Installing " + pluginName + " ");
System.out.println("-> Installing " + pluginName + "...");
try {
pluginManager.downloadPlugin(pluginName);
System.out.println(" DONE");
pluginManager.downloadAndExtract(pluginName);
} catch (IOException e) {
System.out.println("Failed to install " + pluginName + ", reason: " + e.getMessage());
}

28
modules/elasticsearch/src/main/java/org/elasticsearch/plugins/PluginsService.java
View File

@ -23,6 +23,7 @@ import org.elasticsearch.ElasticSearchException;
import org.elasticsearch.common.collect.ImmutableMap;
import org.elasticsearch.common.collect.Lists;
import org.elasticsearch.common.collect.Maps;
import org.elasticsearch.common.collect.Sets;
import org.elasticsearch.common.component.AbstractComponent;
import org.elasticsearch.common.component.LifecycleComponent;
import org.elasticsearch.common.inject.Inject;
@ -37,7 +38,12 @@ import java.io.IOException;
import java.io.InputStream;
import java.lang.reflect.Method;
import java.net.URL;
import java.util.*;
import java.util.Collection;
import java.util.Enumeration;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import static org.elasticsearch.common.collect.Maps.*;
@ -60,7 +66,7 @@ public class PluginsService extends AbstractComponent {
Map<String, Plugin> plugins = Maps.newHashMap();
plugins.putAll(loadPluginsFromClasspath(settings));
logger.info("loaded {}", plugins.keySet());
logger.info("loaded {}, sites {}", plugins.keySet(), sitePlugins());
this.plugins = ImmutableMap.copyOf(plugins);
}
@ -138,6 +144,24 @@ public class PluginsService extends AbstractComponent {
return services;
}
private Set<String> sitePlugins() {
File pluginsFile = environment.pluginsFile();
Set<String> sitePlugins = Sets.newHashSet();
if (!pluginsFile.exists()) {
return sitePlugins;
}
if (!pluginsFile.isDirectory()) {
return sitePlugins;
}
File[] pluginsFiles = pluginsFile.listFiles();
for (File pluginFile : pluginsFiles) {
if (new File(pluginFile, "_site").exists()) {
sitePlugins.add(pluginFile.getName());
}
}
return sitePlugins;
}
private void loadPluginsIntoClassLoader() {
File pluginsFile = environment.pluginsFile();
if (!pluginsFile.exists()) {

40
modules/elasticsearch/src/main/java/org/elasticsearch/http/HttpServerHandler.java → modules/elasticsearch/src/main/java/org/elasticsearch/rest/BytesRestResponse.java
View File

@ -17,14 +17,38 @@
* under the License.
*/
package org.elasticsearch.http;
package org.elasticsearch.rest;
/**
* @author kimchy (Shay Banon)
*/
public interface HttpServerHandler {
import java.io.IOException;
void handleRequest(HttpRequest request, HttpChannel channel);
public class BytesRestResponse extends AbstractRestResponse {
boolean spawn();
}
private final byte[] bytes;
private final String contentType;
public BytesRestResponse(byte[] bytes, String contentType) {
this.bytes = bytes;
this.contentType = contentType;
}
@Override public boolean contentThreadSafe() {
return true;
}
@Override public String contentType() {
return contentType;
}
@Override public byte[] content() throws IOException {
return bytes;
}
@Override public int contentLength() throws IOException {
return bytes.length;
}
@Override public RestStatus status() {
return RestStatus.OK;
}
}