diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000000..9dac7bc0ead --- /dev/null +++ b/.gitignore @@ -0,0 +1,11 @@ +/data +/work +/logs +/.idea +/target +.DS_Store +*.iml +/.settings +/.project +/.classpath +*.vmoptions diff --git a/README.md b/README.md new file mode 100644 index 00000000000..7eb3218e8ab --- /dev/null +++ b/README.md @@ -0,0 +1,263 @@ +Google Compute Engine Cloud Plugin for ElasticSearch +==================================================== + +The GCE Cloud plugin allows to use GCE API for the unicast discovery mechanism. + +In order to install the plugin, simply run: `bin/plugin -install elasticsearch/elasticsearch-cloud-gce/1.0.0-SNAPSHOT`. + + ---------------------------------------------- + | GCE Cloud Plugin | ElasticSearch | + ---------------------------------------------- + | 1.0.0-SNAPSHOT (master) | 0.90 -> master | + ---------------------------------------------- + + + +Google Compute Engine Virtual Machine Discovery +=============================== + +Google Compute Engine VM discovery allows to use the google APIs to perform automatic discovery (similar to multicast in non hostile +multicast environments). Here is a simple sample configuration: + +```yaml + cloud: + gce: + project_id: + zone: + discovery: + type: gce +``` + +How to start (short story) +-------------------------- + +* Create Google Compute Engine instance +* Install Elasticsearch +* Install Google Compute Engine Cloud plugin +* Modify `elasticsearch.yml` file +* Start Elasticsearch + +How to start (long story) +-------------------------- + +### Prerequisites + +Before starting, you should have: + +* [GCUtil](https://developers.google.com/compute/docs/gcutil/#install) +* Your Google service account name. Generally, the account name has the format `123845678986@project.gserviceaccount.com`. +You can get it from [Google APIS Console](https://code.google.com/apis/console/) +* Your project ID. Let's say here `es-cloud`. Get it from [Google APIS Console](https://code.google.com/apis/console/). + + +### Creating your first instance + + +```sh +gcutil --project=es-cloud addinstance myesnode1 --service_account_scope=compute-rw --persistent_boot_disk +``` + +You will be asked to open a link in your browser. Login and allow access to listed services. +You will get back a verification code. Copy and paste it in your terminal. + +You should get `Authentication successful.` message. + +Then, choose your zone. Let's say here that we choose `europe-west1-a`. + +Choose your compute instance size. Let's say `f1-micro`. + +Choose your OS. Let's say `projects/debian-cloud/global/images/debian-7-wheezy-v20130617`. + +You may be asked to create a ssh key. Follow instructions to create one. + +When done, a report like this one should appears: + +```sh +Table of resources: + ++-----------+--------------+-------+---------+--------------+----------------+----------------+----------------+---------+----------------+ +| name | machine-type | image | network | network-ip | external-ip | disks | zone | status | status-message | ++-----------+--------------+-------+---------+--------------+----------------+----------------+----------------+---------+----------------+ +| myesnode1 | f1-micro | | default | 10.240.20.57 | 192.158.29.199 | boot-myesnode1 | europe-west1-a | RUNNING | | ++-----------+--------------+-------+---------+--------------+----------------+----------------+----------------+---------+----------------+ +``` + +Note your external ip address. + +Now, your first instance is started. You need to install Elasticsearch on it. +You can now connect to your machine: + +``` +ssh -i ~/.ssh/google_compute_engine 192.158.29.199 +``` + +Once connected, install Elasticsearch: + +```sh +sudo apt-get update + +# Install curl if needed +sudo apt-get install curl + +# Download Elasticsearch +curl https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-0.90.2.deb -o elasticsearch-0.90.2.deb + +# Prepare Java installation +sudo apt-get install java7-runtime-headless + +# Prepare Elasticsearch installation +sudo dpkg -i elasticsearch-0.90.2.deb +``` + +Check that elasticsearch is running: + +```sh +curl http://localhost:9200/ +``` + +This command should give you a JSON result: + +```javascript +{ + "ok" : true, + "status" : 200, + "name" : "Lunatica", + "version" : { + "number" : "0.90.2", + "snapshot_build" : false + }, + "tagline" : "You Know, for Search" +} +``` + +### Install elasticsearch cloud gce plugin + +Stop elasticsearch: + +```sh +sudo service elasticsearch stop +``` + +Install the plugin: + +```sh +sudo /usr/share/elasticsearch/bin/plugin -install elasticsearch/elasticsearch-cloud-gce/1.0.0-SNAPSHOT +``` + +Configure it: + +```sh +sudo vi /etc/elasticsearch/elasticsearch.yml +``` + +And add the following lines: + +```yaml + cloud: + gce: + project_id: es-cloud + zone: europe-west1-a + discovery: + type: gce +``` + +Restart elasticsearch: + +```sh +sudo service elasticsearch start +``` + +### Cloning your existing machine + +First create an image of your running instance: + +```sh +sudo python /usr/share/imagebundle/image_bundle.py \ + -r / -o /tmp/ --log_file=/tmp/abc.log +``` + +An image has been created in `/tmp` directory: + +```sh +$ ls /tmp +e4686d7f5bf904a924ae0cfeb58d0827c6d5b966.image.tar.gz +``` + +Upload your image to Google Cloud Storage: + +```sh +# Launch this command and follow instructions to give your instance an access to your storage +gsutil config +``` + +Create a bucket to hold your image, let's say `esimage`: + +```sh +gsutil mb gs://esimage +``` + +Copy your image to this bucket: + +```sh +gsutil cp /tmp/e4686d7f5bf904a924ae0cfeb58d0827c6d5b966.image.tar.gz gs://esimage +``` + +Add your image to images collection: + +First, exit from your SSH session: `exit`. + +Then, run locally: + +```sh +gcutil listkernels --project es-cloud ++----------------------------------------------+--------------------------------------------------+-------------+ +| name | description | deprecation | ++----------------------------------------------+--------------------------------------------------+-------------+ +| projects/google/global/kernels/gce-20120621 | 2.6.39-gcg built 2012-03-29 01:07:00 | DEPRECATED | +| projects/google/global/kernels/gce-v20130603 | SCSI-enabled 3.3.8-gcg built 2013-05-29 01:04:00 | | ++----------------------------------------------+--------------------------------------------------+-------------+ +# Note the kernel you prefer to use +gcutil --project=es-cloud addimage elasticsearch-0-90-2 \ + gs://esimage/e4686d7f5bf904a924ae0cfeb58d0827c6d5b966.image.tar.gz \ + --preferred_kernel=projects/google/global/kernels/gce-v20130603 +``` + +Create as many instance as you need: + +```sh +gcutil --project=es-cloud addinstance --image=elasticsearch-0-90-2 \ + --kernel=projects/google/global/kernels/gce-v20130603 myesnode2 \ + --zone europe-west1-a --machine_type f1-micro --service_account_scope=compute-rw \ + --persistent_boot_disk +``` + +### Remove a node + +You can use [Google Cloud Console](https://cloud.google.com/console) or CLI to manage your instances: + +```sh +# Stopping and removing instances +gcutil --project=es-cloud deleteinstance myesnode1 myesnode2 --zone=europe-west1-a + +# Consider removing disk as well if you don't need them anymore +gcutil --project=es-cloud deletedisk boot-myesnode1 boot-myesnode2 --zone=europe-west1-a +``` + +License +------- + +This software is licensed under the Apache 2 license, quoted below. + + Copyright 2009-2013 ElasticSearch + + Licensed under the Apache License, Version 2.0 (the "License"); you may not + use this file except in compliance with the License. You may obtain a copy of + the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations under + the License. diff --git a/pom.xml b/pom.xml new file mode 100644 index 00000000000..4b8c8de3c3a --- /dev/null +++ b/pom.xml @@ -0,0 +1,158 @@ + + + + + GCE cloud plugin + 4.0.0 + org.elasticsearch + elasticsearch-cloud-gce + 1.0.0-SNAPSHOT + + Google Compute Engine Cloud plugin for ElasticSearch + 2013 + + + org.sonatype.oss + oss-parent + 7 + + + + + The Apache Software License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + repo + + + + scm:git:git@github.com:elasticsearch/elasticsearch-cloud-gce.git + scm:git:git@github.com:elasticsearch/elasticsearch-cloud-gce.git + http://github.com/elasticsearch/elasticsearch-cloud-gce + + + + GitHub + https://github.com/elasticsearch/elasticsearch-cloud-gce/issues/ + + + + 0.90.2 + v1beta15-rev3-1.15.0-rc + 1.15.0-rc + + + + + + org.elasticsearch + elasticsearch + ${elasticsearch.version} + + + + + com.google.api-client + google-api-client + ${google.api.version} + + + com.google.http-client + google-http-client-jackson2 + ${google.api.version} + + + com.google.oauth-client + google-oauth-client-jetty + ${google.api.version} + + + com.google.apis + google-api-services-compute + ${google.gce.version} + + + + + log4j + log4j + 1.2.17 + test + + + junit + junit + 4.11 + test + + + + + + + + src/main/resources + true + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.1 + + 1.6 + 1.6 + + + + org.apache.maven.plugins + maven-surefire-plugin + 2.15 + + + org.apache.maven.plugins + maven-source-plugin + 2.2.1 + + + attach-sources + + jar + + + + + + maven-assembly-plugin + 2.4 + + false + ${project.build.directory}/releases/ + + ${basedir}/src/main/assemblies/plugin.xml + + + + + package + + single + + + + + + + diff --git a/src/main/assemblies/plugin.xml b/src/main/assemblies/plugin.xml new file mode 100644 index 00000000000..2a42d629efc --- /dev/null +++ b/src/main/assemblies/plugin.xml @@ -0,0 +1,40 @@ + + + + + plugin + + zip + + false + + + / + true + true + + org.elasticsearch:elasticsearch + + + + / + true + true + + com.google.api-client:google-api-client + com.google.apis:google-api-services-compute + com.google.oauth-client:google-oauth-client-jetty + com.google.http-client:google-http-client-jackson2 + + + + diff --git a/src/main/java/org/elasticsearch/cloud/gce/GceComputeService.java b/src/main/java/org/elasticsearch/cloud/gce/GceComputeService.java new file mode 100644 index 00000000000..8957384bd94 --- /dev/null +++ b/src/main/java/org/elasticsearch/cloud/gce/GceComputeService.java @@ -0,0 +1,222 @@ +/* + * Licensed to ElasticSearch under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.cloud.gce; + +import com.google.api.client.googleapis.compute.ComputeCredential; +import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; +import com.google.api.client.http.HttpTransport; +import com.google.api.client.json.JsonFactory; +import com.google.api.client.json.jackson2.JacksonFactory; +import com.google.api.services.compute.Compute; +import com.google.api.services.compute.model.AccessConfig; +import com.google.api.services.compute.model.Instance; +import com.google.api.services.compute.model.InstanceList; +import com.google.api.services.compute.model.NetworkInterface; +import org.elasticsearch.ElasticSearchException; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.collect.Lists; +import org.elasticsearch.common.component.AbstractLifecycleComponent; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.network.NetworkService; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.settings.SettingsFilter; +import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.discovery.zen.ping.unicast.UnicastZenPing; +import org.elasticsearch.transport.TransportService; + +import java.io.IOException; +import java.net.InetAddress; +import java.security.GeneralSecurityException; +import java.util.List; + +/** + * + */ +public class GceComputeService extends AbstractLifecycleComponent { + + static final class Fields { + private static final String PROJECT = "project_id"; + private static final String ZONE = "zone"; + private static final String VERSION = "Elasticsearch/GceCloud/1.0"; + } + + private List discoNodes; + private TransportService transportService; + private NetworkService networkService; + private Compute compute; + + /** Global instance of the HTTP transport. */ + private static HttpTransport HTTP_TRANSPORT; + + /** Global instance of the JSON factory. */ + private static JsonFactory JSON_FACTORY; + + @Inject + public GceComputeService(Settings settings, SettingsFilter settingsFilter, TransportService transportService, + NetworkService networkService) { + super(settings); + settingsFilter.addFilter(new GceSettingsFilter()); + this.transportService = transportService; + this.networkService = networkService; + } + + /** + * We build the list of Nodes from GCE Management API + * @param project + * @param zone + */ + private List buildNodes(String project, String zone) throws IOException { + + List discoNodes = Lists.newArrayList(); + String ipAddress = null; + try { + InetAddress inetAddress = networkService.resolvePublishHostAddress(null); + if (inetAddress != null) { + ipAddress = inetAddress.getHostAddress(); + } + } catch (IOException e) { + // We can't find the publish host address... Hmmm. Too bad :-( + } + + Compute.Instances.List list = compute.instances().list(project, zone); + + InstanceList instanceList = list.execute(); + + for (Instance instance : instanceList.getItems()) { + String name = instance.getName(); + String type = instance.getMachineType(); + String image = instance.getImage(); + + String ip_public = null; + String ip_private = null; + + List interfaces = instance.getNetworkInterfaces(); + + for (NetworkInterface networkInterface : interfaces) { + if (ip_public == null) { + // Trying to get Public IP Address (For future use) + for (AccessConfig accessConfig : networkInterface.getAccessConfigs()) { + if (Strings.hasText(accessConfig.getNatIP())) { + ip_public = accessConfig.getNatIP(); + break; + } + } + } + + if (ip_private == null) { + ip_private = networkInterface.getNetworkIP(); + } + + // If we have both public and private, we can stop here + if (ip_private != null && ip_public != null) break; + } + + try { + if (ip_private.equals(ipAddress)) { + // We found the current node. + // We can ignore it in the list of DiscoveryNode + logger.debug("current node found. Ignoring {}", ip_private); + } else { + TransportAddress[] addresses = transportService.addressesFromString(ip_private); + // we only limit to 1 addresses, makes no sense to ping 100 ports + for (int i = 0; (i < addresses.length && i < UnicastZenPing.LIMIT_PORTS_COUNT); i++) { + logger.trace("adding {}, address {}, transport_address {}", name, ip_private, addresses[i]); + discoNodes.add(new DiscoveryNode("#cloud-" + name + "-" + i, addresses[i])); + } + } + } catch (Exception e) { + logger.warn("failed to add {}, address {}", e, name, ip_private); + } + + } + + + return discoNodes; + } + + public synchronized List nodes() { + if (this.discoNodes != null) { + return this.discoNodes; + } + try { + HTTP_TRANSPORT = GoogleNetHttpTransport.newTrustedTransport(); + } catch (GeneralSecurityException e) { + + } catch (IOException e) { + + } + JSON_FACTORY = new JacksonFactory(); + + + String project = componentSettings.get(Fields.PROJECT, settings.get("cloud." + Fields.PROJECT)); + String zone = componentSettings.get(Fields.ZONE, settings.get("cloud." + Fields.ZONE)); + + // Check that we have all needed properties + if (!checkProperty(Fields.PROJECT, project)) return null; + if (!checkProperty(Fields.ZONE, zone)) return null; + + try { + logger.debug("starting GCE discovery service for project [{}] on zone [{}]", project, zone); + ComputeCredential credential = new ComputeCredential.Builder(HTTP_TRANSPORT, JSON_FACTORY).build(); + credential.refreshToken(); + + logger.debug("token [{}] will expire in [{}] s", credential.getAccessToken(), credential.getExpiresInSeconds()); + + // Once done, let's use this token + compute = new Compute.Builder(HTTP_TRANSPORT, JSON_FACTORY, null) + .setApplicationName(Fields.VERSION) + .setHttpRequestInitializer(credential) + .build(); + + this.discoNodes = buildNodes(project, zone); + } catch (Throwable t) { + logger.warn("error while trying to find nodes for GCE service [{}]: {}: {}", project, t.getClass().getName(), + t.getMessage()); + logger.debug("error found is: ", t); + // We create an empty list in that specific case. + // So discovery process won't fail with NPE but this node won't join any cluster + this.discoNodes = Lists.newArrayList(); + } + + logger.debug("using dynamic discovery nodes {}", discoNodes); + return this.discoNodes; + } + + @Override + protected void doStart() throws ElasticSearchException { + } + + @Override + protected void doStop() throws ElasticSearchException { + } + + @Override + protected void doClose() throws ElasticSearchException { + } + + private boolean checkProperty(String name, String value) { + if (!Strings.hasText(value)) { + logger.warn("cloud.gce.{} is not set. Disabling gce discovery.", name); + return false; + } + return true; + } +} diff --git a/src/main/java/org/elasticsearch/cloud/gce/GceModule.java b/src/main/java/org/elasticsearch/cloud/gce/GceModule.java new file mode 100644 index 00000000000..938c603bbf7 --- /dev/null +++ b/src/main/java/org/elasticsearch/cloud/gce/GceModule.java @@ -0,0 +1,33 @@ +/* + * Licensed to ElasticSearch under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.cloud.gce; + +import org.elasticsearch.common.inject.AbstractModule; + +/** + * + */ +public class GceModule extends AbstractModule { + + @Override + protected void configure() { + bind(GceComputeService.class).asEagerSingleton(); + } +} diff --git a/src/main/java/org/elasticsearch/cloud/gce/GceSettingsFilter.java b/src/main/java/org/elasticsearch/cloud/gce/GceSettingsFilter.java new file mode 100644 index 00000000000..64aa69a943c --- /dev/null +++ b/src/main/java/org/elasticsearch/cloud/gce/GceSettingsFilter.java @@ -0,0 +1,35 @@ +/* + * Licensed to ElasticSearch under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.cloud.gce; + +import org.elasticsearch.common.settings.ImmutableSettings; +import org.elasticsearch.common.settings.SettingsFilter; + +/** + * + */ +public class GceSettingsFilter implements SettingsFilter.Filter { + + @Override + public void filter(ImmutableSettings.Builder settings) { + settings.remove("cloud.zone"); + settings.remove("cloud.project_id"); + } +} diff --git a/src/main/java/org/elasticsearch/discovery/gce/GceDiscovery.java b/src/main/java/org/elasticsearch/discovery/gce/GceDiscovery.java new file mode 100755 index 00000000000..0907b6d9639 --- /dev/null +++ b/src/main/java/org/elasticsearch/discovery/gce/GceDiscovery.java @@ -0,0 +1,67 @@ +/* + * Licensed to ElasticSearch under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.discovery.gce; + +import org.elasticsearch.cloud.gce.GceComputeService; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.ClusterService; +import org.elasticsearch.cluster.node.DiscoveryNodeService; +import org.elasticsearch.common.collect.ImmutableList; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.discovery.zen.ZenDiscovery; +import org.elasticsearch.discovery.zen.ping.ZenPing; +import org.elasticsearch.discovery.zen.ping.ZenPingService; +import org.elasticsearch.discovery.zen.ping.unicast.UnicastZenPing; +import org.elasticsearch.node.settings.NodeSettingsService; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; + +/** + * + */ +public class GceDiscovery extends ZenDiscovery { + + @Inject + public GceDiscovery(Settings settings, ClusterName clusterName, ThreadPool threadPool, TransportService transportService, + ClusterService clusterService, NodeSettingsService nodeSettingsService, ZenPingService pingService, + DiscoveryNodeService discoveryNodeService, GceComputeService gceComputeService) { + super(settings, clusterName, threadPool, transportService, clusterService, nodeSettingsService, discoveryNodeService, pingService); + if (settings.getAsBoolean("cloud.enabled", true)) { + ImmutableList zenPings = pingService.zenPings(); + UnicastZenPing unicastZenPing = null; + for (ZenPing zenPing : zenPings) { + if (zenPing instanceof UnicastZenPing) { + unicastZenPing = (UnicastZenPing) zenPing; + break; + } + } + + if (unicastZenPing != null) { + // update the unicast zen ping to add cloud hosts provider + // and, while we are at it, use only it and not the multicast for example + unicastZenPing.addHostsProvider(new GceUnicastHostsProvider(settings, gceComputeService.nodes())); + pingService.zenPings(ImmutableList.of(unicastZenPing)); + } else { + logger.warn("failed to apply gce unicast discovery, no unicast ping found"); + } + } + } +} diff --git a/src/main/java/org/elasticsearch/discovery/gce/GceDiscoveryModule.java b/src/main/java/org/elasticsearch/discovery/gce/GceDiscoveryModule.java new file mode 100644 index 00000000000..b7970d9ba97 --- /dev/null +++ b/src/main/java/org/elasticsearch/discovery/gce/GceDiscoveryModule.java @@ -0,0 +1,34 @@ +/* + * Licensed to ElasticSearch under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.discovery.gce; + +import org.elasticsearch.discovery.Discovery; +import org.elasticsearch.discovery.zen.ZenDiscoveryModule; + +/** + * + */ +public class GceDiscoveryModule extends ZenDiscoveryModule { + + @Override + protected void bindDiscovery() { + bind(Discovery.class).to(GceDiscovery.class).asEagerSingleton(); + } +} diff --git a/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java b/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java new file mode 100644 index 00000000000..358a9860c58 --- /dev/null +++ b/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java @@ -0,0 +1,47 @@ +/* + * Licensed to ElasticSearch under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.discovery.gce; + +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.common.component.AbstractComponent; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.discovery.zen.ping.unicast.UnicastHostsProvider; + +import java.util.List; + +/** + * + */ +public class GceUnicastHostsProvider extends AbstractComponent implements UnicastHostsProvider { + + private final List discoNodes; + + @Inject + public GceUnicastHostsProvider(Settings settings, List discoNodes) { + super(settings); + this.discoNodes = discoNodes; + } + + @Override + public List buildDynamicNodes() { + return discoNodes; + } +} diff --git a/src/main/java/org/elasticsearch/plugin/cloud/gce/CloudGcePlugin.java b/src/main/java/org/elasticsearch/plugin/cloud/gce/CloudGcePlugin.java new file mode 100644 index 00000000000..0100a32c6eb --- /dev/null +++ b/src/main/java/org/elasticsearch/plugin/cloud/gce/CloudGcePlugin.java @@ -0,0 +1,71 @@ +/* + * Licensed to ElasticSearch under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.plugin.cloud.gce; + +import org.elasticsearch.cloud.gce.GceComputeService; +import org.elasticsearch.cloud.gce.GceModule; +import org.elasticsearch.common.collect.Lists; +import org.elasticsearch.common.component.LifecycleComponent; +import org.elasticsearch.common.inject.Module; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.plugins.AbstractPlugin; + +import java.util.Collection; + +/** + * + */ +public class CloudGcePlugin extends AbstractPlugin { + + private final Settings settings; + + public CloudGcePlugin(Settings settings) { + this.settings = settings; + } + + @Override + public String name() { + return "cloud-gce"; + } + + @Override + public String description() { + return "Cloud Google Compute Engine Plugin"; + } + + @Override + public Collection> modules() { + Collection> modules = Lists.newArrayList(); + if (settings.getAsBoolean("cloud.enabled", true)) { + modules.add(GceModule.class); + } + return modules; + } + + @Override + public Collection> services() { + Collection> services = Lists.newArrayList(); + if (settings.getAsBoolean("cloud.enabled", true)) { + services.add(GceComputeService.class); + } + return services; + } + +} diff --git a/src/main/resources/es-plugin.properties b/src/main/resources/es-plugin.properties new file mode 100644 index 00000000000..e6dfb1b8d78 --- /dev/null +++ b/src/main/resources/es-plugin.properties @@ -0,0 +1,12 @@ +# Licensed to ElasticSearch under one or more contributor +# license agreements. See the NOTICE file distributed with this work for additional +# information regarding copyright ownership. ElasticSearch licenses this file to you +# under the Apache License, Version 2.0 (the "License"); you may not use this +# file except in compliance with the License. You may obtain a copy of the +# License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by +# applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the specific language +# governing permissions and limitations under the License. +plugin=org.elasticsearch.plugin.cloud.gce.CloudGcePlugin +version=${project.version} diff --git a/src/test/java/org/elasticsearch/cloud/gce/tests/GceSimpleTest.java b/src/test/java/org/elasticsearch/cloud/gce/tests/GceSimpleTest.java new file mode 100644 index 00000000000..90fcead8ccd --- /dev/null +++ b/src/test/java/org/elasticsearch/cloud/gce/tests/GceSimpleTest.java @@ -0,0 +1,60 @@ +/* + * Licensed to ElasticSearch under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.cloud.gce.tests; + +import org.elasticsearch.common.io.FileSystemUtils; +import org.elasticsearch.common.settings.ImmutableSettings; +import org.elasticsearch.node.Node; +import org.elasticsearch.node.NodeBuilder; +import org.junit.Ignore; +import org.junit.Test; + +import java.io.File; + +/** + * This test can't work on a local machine as it needs to be launched from + * a GCE instance + * TODO: Mock GCE to simulate APIs REST Responses + */ +@Ignore +public class GceSimpleTest { + + @Test + public void launchNode() { + File dataDir = new File("./target/es/data"); + if(dataDir.exists()) { + FileSystemUtils.deleteRecursively(dataDir, true); + } + + // Then we start our node for tests + Node node = NodeBuilder + .nodeBuilder() + .settings( + ImmutableSettings.settingsBuilder() + .put("gateway.type", "local") + .put("path.data", "./target/es/data") + .put("path.logs", "./target/es/logs") + .put("path.work", "./target/es/work") + ).node(); + + // We wait now for the yellow (or green) status +// node.client().admin().cluster().prepareHealth().setWaitForYellowStatus().execute().actionGet(); + + } +} diff --git a/src/test/resources/elasticsearch.yml b/src/test/resources/elasticsearch.yml new file mode 100644 index 00000000000..c50aef0d2e2 --- /dev/null +++ b/src/test/resources/elasticsearch.yml @@ -0,0 +1,26 @@ +# Licensed to ElasticSearch under one or more contributor +# license agreements. See the NOTICE file distributed with this work for additional +# information regarding copyright ownership. ElasticSearch licenses this file to you +# under the Apache License, Version 2.0 (the "License"); you may not use this +# file except in compliance with the License. You may obtain a copy of the +# License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by +# applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the specific language +# governing permissions and limitations under the License. + + cluster.name: gce +# GCE discovery allows to use GCE API in order to perform discovery. +# +# You have to install the cloud-gce plugin for enabling the GCE discovery. +# +# See +# for more information. +# +# See README in elasticsearch-cloud-gce repository for a step-by-step tutorial. + cloud: + gce: + project_id: es-cloud + zone: europe-west1-a + discovery: + type: gce diff --git a/src/test/resources/log4j.xml b/src/test/resources/log4j.xml new file mode 100644 index 00000000000..46249c145c5 --- /dev/null +++ b/src/test/resources/log4j.xml @@ -0,0 +1,39 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +