From 222b36344f0939f12cf54358e7047239cd398463 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Fri, 28 Jun 2013 09:22:10 +0200 Subject: [PATCH 01/96] Initial commit --- LICENSE.txt | 202 ++++++++++++++++++++++++++++++++++++++++++++++++++++ NOTICE.txt | 5 ++ 2 files changed, 207 insertions(+) create mode 100644 LICENSE.txt create mode 100644 NOTICE.txt diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 00000000000..d6456956733 --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/NOTICE.txt b/NOTICE.txt new file mode 100644 index 00000000000..86016475acd --- /dev/null +++ b/NOTICE.txt @@ -0,0 +1,5 @@ +ElasticSearch +Copyright 2009-2013 ElasticSearch + +This product includes software developed by The Apache Software +Foundation (http://www.apache.org/). From 5e141479918ed0e6a1dd8bc4293d68fd2e819de9 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Fri, 28 Jun 2013 09:45:02 +0200 Subject: [PATCH 02/96] Add cloud discovery for google compute engine Google Compute Engine VM discovery allows to use the google APIs to perform automatic discovery (similar to multicast in non hostile multicast environments). Here is a simple sample configuration: ```yaml cloud: gce: project_id: zone: discovery: type: gce ``` How to start (short story) -------------------------- * Create Google Compute Engine instance * Install Elasticsearch * Install Google Compute Engine Cloud plugin * Modify `elasticsearch.yml` file * Start Elasticsearch Closes #1. --- .gitignore | 11 + README.md | 263 ++++++++++++++++++ pom.xml | 158 +++++++++++ src/main/assemblies/plugin.xml | 40 +++ .../cloud/gce/GceComputeService.java | 222 +++++++++++++++ .../elasticsearch/cloud/gce/GceModule.java | 33 +++ .../cloud/gce/GceSettingsFilter.java | 35 +++ .../discovery/gce/GceDiscovery.java | 67 +++++ .../discovery/gce/GceDiscoveryModule.java | 34 +++ .../gce/GceUnicastHostsProvider.java | 47 ++++ .../plugin/cloud/gce/CloudGcePlugin.java | 71 +++++ src/main/resources/es-plugin.properties | 12 + .../cloud/gce/tests/GceSimpleTest.java | 60 ++++ src/test/resources/elasticsearch.yml | 26 ++ src/test/resources/log4j.xml | 39 +++ 15 files changed, 1118 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100644 pom.xml create mode 100644 src/main/assemblies/plugin.xml create mode 100644 src/main/java/org/elasticsearch/cloud/gce/GceComputeService.java create mode 100644 src/main/java/org/elasticsearch/cloud/gce/GceModule.java create mode 100644 src/main/java/org/elasticsearch/cloud/gce/GceSettingsFilter.java create mode 100755 src/main/java/org/elasticsearch/discovery/gce/GceDiscovery.java create mode 100644 src/main/java/org/elasticsearch/discovery/gce/GceDiscoveryModule.java create mode 100644 src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java create mode 100644 src/main/java/org/elasticsearch/plugin/cloud/gce/CloudGcePlugin.java create mode 100644 src/main/resources/es-plugin.properties create mode 100644 src/test/java/org/elasticsearch/cloud/gce/tests/GceSimpleTest.java create mode 100644 src/test/resources/elasticsearch.yml create mode 100644 src/test/resources/log4j.xml diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000000..9dac7bc0ead --- /dev/null +++ b/.gitignore @@ -0,0 +1,11 @@ +/data +/work +/logs +/.idea +/target +.DS_Store +*.iml +/.settings +/.project +/.classpath +*.vmoptions diff --git a/README.md b/README.md new file mode 100644 index 00000000000..7eb3218e8ab --- /dev/null +++ b/README.md @@ -0,0 +1,263 @@ +Google Compute Engine Cloud Plugin for ElasticSearch +==================================================== + +The GCE Cloud plugin allows to use GCE API for the unicast discovery mechanism. + +In order to install the plugin, simply run: `bin/plugin -install elasticsearch/elasticsearch-cloud-gce/1.0.0-SNAPSHOT`. + + ---------------------------------------------- + | GCE Cloud Plugin | ElasticSearch | + ---------------------------------------------- + | 1.0.0-SNAPSHOT (master) | 0.90 -> master | + ---------------------------------------------- + + + +Google Compute Engine Virtual Machine Discovery +=============================== + +Google Compute Engine VM discovery allows to use the google APIs to perform automatic discovery (similar to multicast in non hostile +multicast environments). Here is a simple sample configuration: + +```yaml + cloud: + gce: + project_id: + zone: + discovery: + type: gce +``` + +How to start (short story) +-------------------------- + +* Create Google Compute Engine instance +* Install Elasticsearch +* Install Google Compute Engine Cloud plugin +* Modify `elasticsearch.yml` file +* Start Elasticsearch + +How to start (long story) +-------------------------- + +### Prerequisites + +Before starting, you should have: + +* [GCUtil](https://developers.google.com/compute/docs/gcutil/#install) +* Your Google service account name. Generally, the account name has the format `123845678986@project.gserviceaccount.com`. +You can get it from [Google APIS Console](https://code.google.com/apis/console/) +* Your project ID. Let's say here `es-cloud`. Get it from [Google APIS Console](https://code.google.com/apis/console/). + + +### Creating your first instance + + +```sh +gcutil --project=es-cloud addinstance myesnode1 --service_account_scope=compute-rw --persistent_boot_disk +``` + +You will be asked to open a link in your browser. Login and allow access to listed services. +You will get back a verification code. Copy and paste it in your terminal. + +You should get `Authentication successful.` message. + +Then, choose your zone. Let's say here that we choose `europe-west1-a`. + +Choose your compute instance size. Let's say `f1-micro`. + +Choose your OS. Let's say `projects/debian-cloud/global/images/debian-7-wheezy-v20130617`. + +You may be asked to create a ssh key. Follow instructions to create one. + +When done, a report like this one should appears: + +```sh +Table of resources: + ++-----------+--------------+-------+---------+--------------+----------------+----------------+----------------+---------+----------------+ +| name | machine-type | image | network | network-ip | external-ip | disks | zone | status | status-message | ++-----------+--------------+-------+---------+--------------+----------------+----------------+----------------+---------+----------------+ +| myesnode1 | f1-micro | | default | 10.240.20.57 | 192.158.29.199 | boot-myesnode1 | europe-west1-a | RUNNING | | ++-----------+--------------+-------+---------+--------------+----------------+----------------+----------------+---------+----------------+ +``` + +Note your external ip address. + +Now, your first instance is started. You need to install Elasticsearch on it. +You can now connect to your machine: + +``` +ssh -i ~/.ssh/google_compute_engine 192.158.29.199 +``` + +Once connected, install Elasticsearch: + +```sh +sudo apt-get update + +# Install curl if needed +sudo apt-get install curl + +# Download Elasticsearch +curl https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-0.90.2.deb -o elasticsearch-0.90.2.deb + +# Prepare Java installation +sudo apt-get install java7-runtime-headless + +# Prepare Elasticsearch installation +sudo dpkg -i elasticsearch-0.90.2.deb +``` + +Check that elasticsearch is running: + +```sh +curl http://localhost:9200/ +``` + +This command should give you a JSON result: + +```javascript +{ + "ok" : true, + "status" : 200, + "name" : "Lunatica", + "version" : { + "number" : "0.90.2", + "snapshot_build" : false + }, + "tagline" : "You Know, for Search" +} +``` + +### Install elasticsearch cloud gce plugin + +Stop elasticsearch: + +```sh +sudo service elasticsearch stop +``` + +Install the plugin: + +```sh +sudo /usr/share/elasticsearch/bin/plugin -install elasticsearch/elasticsearch-cloud-gce/1.0.0-SNAPSHOT +``` + +Configure it: + +```sh +sudo vi /etc/elasticsearch/elasticsearch.yml +``` + +And add the following lines: + +```yaml + cloud: + gce: + project_id: es-cloud + zone: europe-west1-a + discovery: + type: gce +``` + +Restart elasticsearch: + +```sh +sudo service elasticsearch start +``` + +### Cloning your existing machine + +First create an image of your running instance: + +```sh +sudo python /usr/share/imagebundle/image_bundle.py \ + -r / -o /tmp/ --log_file=/tmp/abc.log +``` + +An image has been created in `/tmp` directory: + +```sh +$ ls /tmp +e4686d7f5bf904a924ae0cfeb58d0827c6d5b966.image.tar.gz +``` + +Upload your image to Google Cloud Storage: + +```sh +# Launch this command and follow instructions to give your instance an access to your storage +gsutil config +``` + +Create a bucket to hold your image, let's say `esimage`: + +```sh +gsutil mb gs://esimage +``` + +Copy your image to this bucket: + +```sh +gsutil cp /tmp/e4686d7f5bf904a924ae0cfeb58d0827c6d5b966.image.tar.gz gs://esimage +``` + +Add your image to images collection: + +First, exit from your SSH session: `exit`. + +Then, run locally: + +```sh +gcutil listkernels --project es-cloud ++----------------------------------------------+--------------------------------------------------+-------------+ +| name | description | deprecation | ++----------------------------------------------+--------------------------------------------------+-------------+ +| projects/google/global/kernels/gce-20120621 | 2.6.39-gcg built 2012-03-29 01:07:00 | DEPRECATED | +| projects/google/global/kernels/gce-v20130603 | SCSI-enabled 3.3.8-gcg built 2013-05-29 01:04:00 | | ++----------------------------------------------+--------------------------------------------------+-------------+ +# Note the kernel you prefer to use +gcutil --project=es-cloud addimage elasticsearch-0-90-2 \ + gs://esimage/e4686d7f5bf904a924ae0cfeb58d0827c6d5b966.image.tar.gz \ + --preferred_kernel=projects/google/global/kernels/gce-v20130603 +``` + +Create as many instance as you need: + +```sh +gcutil --project=es-cloud addinstance --image=elasticsearch-0-90-2 \ + --kernel=projects/google/global/kernels/gce-v20130603 myesnode2 \ + --zone europe-west1-a --machine_type f1-micro --service_account_scope=compute-rw \ + --persistent_boot_disk +``` + +### Remove a node + +You can use [Google Cloud Console](https://cloud.google.com/console) or CLI to manage your instances: + +```sh +# Stopping and removing instances +gcutil --project=es-cloud deleteinstance myesnode1 myesnode2 --zone=europe-west1-a + +# Consider removing disk as well if you don't need them anymore +gcutil --project=es-cloud deletedisk boot-myesnode1 boot-myesnode2 --zone=europe-west1-a +``` + +License +------- + +This software is licensed under the Apache 2 license, quoted below. + + Copyright 2009-2013 ElasticSearch + + Licensed under the Apache License, Version 2.0 (the "License"); you may not + use this file except in compliance with the License. You may obtain a copy of + the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations under + the License. diff --git a/pom.xml b/pom.xml new file mode 100644 index 00000000000..4b8c8de3c3a --- /dev/null +++ b/pom.xml @@ -0,0 +1,158 @@ + + + + + GCE cloud plugin + 4.0.0 + org.elasticsearch + elasticsearch-cloud-gce + 1.0.0-SNAPSHOT + + Google Compute Engine Cloud plugin for ElasticSearch + 2013 + + + org.sonatype.oss + oss-parent + 7 + + + + + The Apache Software License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + repo + + + + scm:git:git@github.com:elasticsearch/elasticsearch-cloud-gce.git + scm:git:git@github.com:elasticsearch/elasticsearch-cloud-gce.git + http://github.com/elasticsearch/elasticsearch-cloud-gce + + + + GitHub + https://github.com/elasticsearch/elasticsearch-cloud-gce/issues/ + + + + 0.90.2 + v1beta15-rev3-1.15.0-rc + 1.15.0-rc + + + + + + org.elasticsearch + elasticsearch + ${elasticsearch.version} + + + + + com.google.api-client + google-api-client + ${google.api.version} + + + com.google.http-client + google-http-client-jackson2 + ${google.api.version} + + + com.google.oauth-client + google-oauth-client-jetty + ${google.api.version} + + + com.google.apis + google-api-services-compute + ${google.gce.version} + + + + + log4j + log4j + 1.2.17 + test + + + junit + junit + 4.11 + test + + + + + + + + src/main/resources + true + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.1 + + 1.6 + 1.6 + + + + org.apache.maven.plugins + maven-surefire-plugin + 2.15 + + + org.apache.maven.plugins + maven-source-plugin + 2.2.1 + + + attach-sources + + jar + + + + + + maven-assembly-plugin + 2.4 + + false + ${project.build.directory}/releases/ + + ${basedir}/src/main/assemblies/plugin.xml + + + + + package + + single + + + + + + + diff --git a/src/main/assemblies/plugin.xml b/src/main/assemblies/plugin.xml new file mode 100644 index 00000000000..2a42d629efc --- /dev/null +++ b/src/main/assemblies/plugin.xml @@ -0,0 +1,40 @@ + + + + + plugin + + zip + + false + + + / + true + true + + org.elasticsearch:elasticsearch + + + + / + true + true + + com.google.api-client:google-api-client + com.google.apis:google-api-services-compute + com.google.oauth-client:google-oauth-client-jetty + com.google.http-client:google-http-client-jackson2 + + + + diff --git a/src/main/java/org/elasticsearch/cloud/gce/GceComputeService.java b/src/main/java/org/elasticsearch/cloud/gce/GceComputeService.java new file mode 100644 index 00000000000..8957384bd94 --- /dev/null +++ b/src/main/java/org/elasticsearch/cloud/gce/GceComputeService.java @@ -0,0 +1,222 @@ +/* + * Licensed to ElasticSearch under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.cloud.gce; + +import com.google.api.client.googleapis.compute.ComputeCredential; +import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; +import com.google.api.client.http.HttpTransport; +import com.google.api.client.json.JsonFactory; +import com.google.api.client.json.jackson2.JacksonFactory; +import com.google.api.services.compute.Compute; +import com.google.api.services.compute.model.AccessConfig; +import com.google.api.services.compute.model.Instance; +import com.google.api.services.compute.model.InstanceList; +import com.google.api.services.compute.model.NetworkInterface; +import org.elasticsearch.ElasticSearchException; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.collect.Lists; +import org.elasticsearch.common.component.AbstractLifecycleComponent; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.network.NetworkService; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.settings.SettingsFilter; +import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.discovery.zen.ping.unicast.UnicastZenPing; +import org.elasticsearch.transport.TransportService; + +import java.io.IOException; +import java.net.InetAddress; +import java.security.GeneralSecurityException; +import java.util.List; + +/** + * + */ +public class GceComputeService extends AbstractLifecycleComponent { + + static final class Fields { + private static final String PROJECT = "project_id"; + private static final String ZONE = "zone"; + private static final String VERSION = "Elasticsearch/GceCloud/1.0"; + } + + private List discoNodes; + private TransportService transportService; + private NetworkService networkService; + private Compute compute; + + /** Global instance of the HTTP transport. */ + private static HttpTransport HTTP_TRANSPORT; + + /** Global instance of the JSON factory. */ + private static JsonFactory JSON_FACTORY; + + @Inject + public GceComputeService(Settings settings, SettingsFilter settingsFilter, TransportService transportService, + NetworkService networkService) { + super(settings); + settingsFilter.addFilter(new GceSettingsFilter()); + this.transportService = transportService; + this.networkService = networkService; + } + + /** + * We build the list of Nodes from GCE Management API + * @param project + * @param zone + */ + private List buildNodes(String project, String zone) throws IOException { + + List discoNodes = Lists.newArrayList(); + String ipAddress = null; + try { + InetAddress inetAddress = networkService.resolvePublishHostAddress(null); + if (inetAddress != null) { + ipAddress = inetAddress.getHostAddress(); + } + } catch (IOException e) { + // We can't find the publish host address... Hmmm. Too bad :-( + } + + Compute.Instances.List list = compute.instances().list(project, zone); + + InstanceList instanceList = list.execute(); + + for (Instance instance : instanceList.getItems()) { + String name = instance.getName(); + String type = instance.getMachineType(); + String image = instance.getImage(); + + String ip_public = null; + String ip_private = null; + + List interfaces = instance.getNetworkInterfaces(); + + for (NetworkInterface networkInterface : interfaces) { + if (ip_public == null) { + // Trying to get Public IP Address (For future use) + for (AccessConfig accessConfig : networkInterface.getAccessConfigs()) { + if (Strings.hasText(accessConfig.getNatIP())) { + ip_public = accessConfig.getNatIP(); + break; + } + } + } + + if (ip_private == null) { + ip_private = networkInterface.getNetworkIP(); + } + + // If we have both public and private, we can stop here + if (ip_private != null && ip_public != null) break; + } + + try { + if (ip_private.equals(ipAddress)) { + // We found the current node. + // We can ignore it in the list of DiscoveryNode + logger.debug("current node found. Ignoring {}", ip_private); + } else { + TransportAddress[] addresses = transportService.addressesFromString(ip_private); + // we only limit to 1 addresses, makes no sense to ping 100 ports + for (int i = 0; (i < addresses.length && i < UnicastZenPing.LIMIT_PORTS_COUNT); i++) { + logger.trace("adding {}, address {}, transport_address {}", name, ip_private, addresses[i]); + discoNodes.add(new DiscoveryNode("#cloud-" + name + "-" + i, addresses[i])); + } + } + } catch (Exception e) { + logger.warn("failed to add {}, address {}", e, name, ip_private); + } + + } + + + return discoNodes; + } + + public synchronized List nodes() { + if (this.discoNodes != null) { + return this.discoNodes; + } + try { + HTTP_TRANSPORT = GoogleNetHttpTransport.newTrustedTransport(); + } catch (GeneralSecurityException e) { + + } catch (IOException e) { + + } + JSON_FACTORY = new JacksonFactory(); + + + String project = componentSettings.get(Fields.PROJECT, settings.get("cloud." + Fields.PROJECT)); + String zone = componentSettings.get(Fields.ZONE, settings.get("cloud." + Fields.ZONE)); + + // Check that we have all needed properties + if (!checkProperty(Fields.PROJECT, project)) return null; + if (!checkProperty(Fields.ZONE, zone)) return null; + + try { + logger.debug("starting GCE discovery service for project [{}] on zone [{}]", project, zone); + ComputeCredential credential = new ComputeCredential.Builder(HTTP_TRANSPORT, JSON_FACTORY).build(); + credential.refreshToken(); + + logger.debug("token [{}] will expire in [{}] s", credential.getAccessToken(), credential.getExpiresInSeconds()); + + // Once done, let's use this token + compute = new Compute.Builder(HTTP_TRANSPORT, JSON_FACTORY, null) + .setApplicationName(Fields.VERSION) + .setHttpRequestInitializer(credential) + .build(); + + this.discoNodes = buildNodes(project, zone); + } catch (Throwable t) { + logger.warn("error while trying to find nodes for GCE service [{}]: {}: {}", project, t.getClass().getName(), + t.getMessage()); + logger.debug("error found is: ", t); + // We create an empty list in that specific case. + // So discovery process won't fail with NPE but this node won't join any cluster + this.discoNodes = Lists.newArrayList(); + } + + logger.debug("using dynamic discovery nodes {}", discoNodes); + return this.discoNodes; + } + + @Override + protected void doStart() throws ElasticSearchException { + } + + @Override + protected void doStop() throws ElasticSearchException { + } + + @Override + protected void doClose() throws ElasticSearchException { + } + + private boolean checkProperty(String name, String value) { + if (!Strings.hasText(value)) { + logger.warn("cloud.gce.{} is not set. Disabling gce discovery.", name); + return false; + } + return true; + } +} diff --git a/src/main/java/org/elasticsearch/cloud/gce/GceModule.java b/src/main/java/org/elasticsearch/cloud/gce/GceModule.java new file mode 100644 index 00000000000..938c603bbf7 --- /dev/null +++ b/src/main/java/org/elasticsearch/cloud/gce/GceModule.java @@ -0,0 +1,33 @@ +/* + * Licensed to ElasticSearch under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.cloud.gce; + +import org.elasticsearch.common.inject.AbstractModule; + +/** + * + */ +public class GceModule extends AbstractModule { + + @Override + protected void configure() { + bind(GceComputeService.class).asEagerSingleton(); + } +} diff --git a/src/main/java/org/elasticsearch/cloud/gce/GceSettingsFilter.java b/src/main/java/org/elasticsearch/cloud/gce/GceSettingsFilter.java new file mode 100644 index 00000000000..64aa69a943c --- /dev/null +++ b/src/main/java/org/elasticsearch/cloud/gce/GceSettingsFilter.java @@ -0,0 +1,35 @@ +/* + * Licensed to ElasticSearch under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.cloud.gce; + +import org.elasticsearch.common.settings.ImmutableSettings; +import org.elasticsearch.common.settings.SettingsFilter; + +/** + * + */ +public class GceSettingsFilter implements SettingsFilter.Filter { + + @Override + public void filter(ImmutableSettings.Builder settings) { + settings.remove("cloud.zone"); + settings.remove("cloud.project_id"); + } +} diff --git a/src/main/java/org/elasticsearch/discovery/gce/GceDiscovery.java b/src/main/java/org/elasticsearch/discovery/gce/GceDiscovery.java new file mode 100755 index 00000000000..0907b6d9639 --- /dev/null +++ b/src/main/java/org/elasticsearch/discovery/gce/GceDiscovery.java @@ -0,0 +1,67 @@ +/* + * Licensed to ElasticSearch under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.discovery.gce; + +import org.elasticsearch.cloud.gce.GceComputeService; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.ClusterService; +import org.elasticsearch.cluster.node.DiscoveryNodeService; +import org.elasticsearch.common.collect.ImmutableList; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.discovery.zen.ZenDiscovery; +import org.elasticsearch.discovery.zen.ping.ZenPing; +import org.elasticsearch.discovery.zen.ping.ZenPingService; +import org.elasticsearch.discovery.zen.ping.unicast.UnicastZenPing; +import org.elasticsearch.node.settings.NodeSettingsService; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; + +/** + * + */ +public class GceDiscovery extends ZenDiscovery { + + @Inject + public GceDiscovery(Settings settings, ClusterName clusterName, ThreadPool threadPool, TransportService transportService, + ClusterService clusterService, NodeSettingsService nodeSettingsService, ZenPingService pingService, + DiscoveryNodeService discoveryNodeService, GceComputeService gceComputeService) { + super(settings, clusterName, threadPool, transportService, clusterService, nodeSettingsService, discoveryNodeService, pingService); + if (settings.getAsBoolean("cloud.enabled", true)) { + ImmutableList zenPings = pingService.zenPings(); + UnicastZenPing unicastZenPing = null; + for (ZenPing zenPing : zenPings) { + if (zenPing instanceof UnicastZenPing) { + unicastZenPing = (UnicastZenPing) zenPing; + break; + } + } + + if (unicastZenPing != null) { + // update the unicast zen ping to add cloud hosts provider + // and, while we are at it, use only it and not the multicast for example + unicastZenPing.addHostsProvider(new GceUnicastHostsProvider(settings, gceComputeService.nodes())); + pingService.zenPings(ImmutableList.of(unicastZenPing)); + } else { + logger.warn("failed to apply gce unicast discovery, no unicast ping found"); + } + } + } +} diff --git a/src/main/java/org/elasticsearch/discovery/gce/GceDiscoveryModule.java b/src/main/java/org/elasticsearch/discovery/gce/GceDiscoveryModule.java new file mode 100644 index 00000000000..b7970d9ba97 --- /dev/null +++ b/src/main/java/org/elasticsearch/discovery/gce/GceDiscoveryModule.java @@ -0,0 +1,34 @@ +/* + * Licensed to ElasticSearch under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.discovery.gce; + +import org.elasticsearch.discovery.Discovery; +import org.elasticsearch.discovery.zen.ZenDiscoveryModule; + +/** + * + */ +public class GceDiscoveryModule extends ZenDiscoveryModule { + + @Override + protected void bindDiscovery() { + bind(Discovery.class).to(GceDiscovery.class).asEagerSingleton(); + } +} diff --git a/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java b/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java new file mode 100644 index 00000000000..358a9860c58 --- /dev/null +++ b/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java @@ -0,0 +1,47 @@ +/* + * Licensed to ElasticSearch under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.discovery.gce; + +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.common.component.AbstractComponent; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.discovery.zen.ping.unicast.UnicastHostsProvider; + +import java.util.List; + +/** + * + */ +public class GceUnicastHostsProvider extends AbstractComponent implements UnicastHostsProvider { + + private final List discoNodes; + + @Inject + public GceUnicastHostsProvider(Settings settings, List discoNodes) { + super(settings); + this.discoNodes = discoNodes; + } + + @Override + public List buildDynamicNodes() { + return discoNodes; + } +} diff --git a/src/main/java/org/elasticsearch/plugin/cloud/gce/CloudGcePlugin.java b/src/main/java/org/elasticsearch/plugin/cloud/gce/CloudGcePlugin.java new file mode 100644 index 00000000000..0100a32c6eb --- /dev/null +++ b/src/main/java/org/elasticsearch/plugin/cloud/gce/CloudGcePlugin.java @@ -0,0 +1,71 @@ +/* + * Licensed to ElasticSearch under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.plugin.cloud.gce; + +import org.elasticsearch.cloud.gce.GceComputeService; +import org.elasticsearch.cloud.gce.GceModule; +import org.elasticsearch.common.collect.Lists; +import org.elasticsearch.common.component.LifecycleComponent; +import org.elasticsearch.common.inject.Module; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.plugins.AbstractPlugin; + +import java.util.Collection; + +/** + * + */ +public class CloudGcePlugin extends AbstractPlugin { + + private final Settings settings; + + public CloudGcePlugin(Settings settings) { + this.settings = settings; + } + + @Override + public String name() { + return "cloud-gce"; + } + + @Override + public String description() { + return "Cloud Google Compute Engine Plugin"; + } + + @Override + public Collection> modules() { + Collection> modules = Lists.newArrayList(); + if (settings.getAsBoolean("cloud.enabled", true)) { + modules.add(GceModule.class); + } + return modules; + } + + @Override + public Collection> services() { + Collection> services = Lists.newArrayList(); + if (settings.getAsBoolean("cloud.enabled", true)) { + services.add(GceComputeService.class); + } + return services; + } + +} diff --git a/src/main/resources/es-plugin.properties b/src/main/resources/es-plugin.properties new file mode 100644 index 00000000000..e6dfb1b8d78 --- /dev/null +++ b/src/main/resources/es-plugin.properties @@ -0,0 +1,12 @@ +# Licensed to ElasticSearch under one or more contributor +# license agreements. See the NOTICE file distributed with this work for additional +# information regarding copyright ownership. ElasticSearch licenses this file to you +# under the Apache License, Version 2.0 (the "License"); you may not use this +# file except in compliance with the License. You may obtain a copy of the +# License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by +# applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the specific language +# governing permissions and limitations under the License. +plugin=org.elasticsearch.plugin.cloud.gce.CloudGcePlugin +version=${project.version} diff --git a/src/test/java/org/elasticsearch/cloud/gce/tests/GceSimpleTest.java b/src/test/java/org/elasticsearch/cloud/gce/tests/GceSimpleTest.java new file mode 100644 index 00000000000..90fcead8ccd --- /dev/null +++ b/src/test/java/org/elasticsearch/cloud/gce/tests/GceSimpleTest.java @@ -0,0 +1,60 @@ +/* + * Licensed to ElasticSearch under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.cloud.gce.tests; + +import org.elasticsearch.common.io.FileSystemUtils; +import org.elasticsearch.common.settings.ImmutableSettings; +import org.elasticsearch.node.Node; +import org.elasticsearch.node.NodeBuilder; +import org.junit.Ignore; +import org.junit.Test; + +import java.io.File; + +/** + * This test can't work on a local machine as it needs to be launched from + * a GCE instance + * TODO: Mock GCE to simulate APIs REST Responses + */ +@Ignore +public class GceSimpleTest { + + @Test + public void launchNode() { + File dataDir = new File("./target/es/data"); + if(dataDir.exists()) { + FileSystemUtils.deleteRecursively(dataDir, true); + } + + // Then we start our node for tests + Node node = NodeBuilder + .nodeBuilder() + .settings( + ImmutableSettings.settingsBuilder() + .put("gateway.type", "local") + .put("path.data", "./target/es/data") + .put("path.logs", "./target/es/logs") + .put("path.work", "./target/es/work") + ).node(); + + // We wait now for the yellow (or green) status +// node.client().admin().cluster().prepareHealth().setWaitForYellowStatus().execute().actionGet(); + + } +} diff --git a/src/test/resources/elasticsearch.yml b/src/test/resources/elasticsearch.yml new file mode 100644 index 00000000000..c50aef0d2e2 --- /dev/null +++ b/src/test/resources/elasticsearch.yml @@ -0,0 +1,26 @@ +# Licensed to ElasticSearch under one or more contributor +# license agreements. See the NOTICE file distributed with this work for additional +# information regarding copyright ownership. ElasticSearch licenses this file to you +# under the Apache License, Version 2.0 (the "License"); you may not use this +# file except in compliance with the License. You may obtain a copy of the +# License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by +# applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the specific language +# governing permissions and limitations under the License. + + cluster.name: gce +# GCE discovery allows to use GCE API in order to perform discovery. +# +# You have to install the cloud-gce plugin for enabling the GCE discovery. +# +# See +# for more information. +# +# See README in elasticsearch-cloud-gce repository for a step-by-step tutorial. + cloud: + gce: + project_id: es-cloud + zone: europe-west1-a + discovery: + type: gce diff --git a/src/test/resources/log4j.xml b/src/test/resources/log4j.xml new file mode 100644 index 00000000000..46249c145c5 --- /dev/null +++ b/src/test/resources/log4j.xml @@ -0,0 +1,39 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From 703c2b3e828beac2b570b36a8fa6cfba8936940a Mon Sep 17 00:00:00 2001 From: David Pilato Date: Fri, 28 Jun 2013 15:44:52 +0200 Subject: [PATCH 03/96] Filtering by tags ----------------- The GCE discovery can also filter machines to include in the cluster based on tags using `discovery.gce.tags` settings. For example, setting `discovery.gce.tags` to `dev` will only filter instances having a tag set to `dev`. Several tags set will require all of those tags to be set for the instance to be included. One practical use for tag filtering is when an GCE cluster contains many nodes that are not running elasticsearch. In this case (particularly with high ping_timeout values) there is a risk that a new node's discovery phase will end before it has found the cluster (which will result in it declaring itself master of a new cluster with the same name - highly undesirable). Adding tag on elasticsearch GCE nodes and then filtering by that tag will resolve this issue. Add your tag when building the new instance: ```sh gcutil --project=es-cloud addinstance myesnode1 --service_account_scope=compute-rw --persistent_boot_disk \ --tags=elasticsearch,dev ``` Then, define it in `elasticsearch.yml`: ```yaml cloud: gce: project_id: es-cloud zone: europe-west1-a discovery: type: gce gce: tags: elasticsearch, dev ``` Closes #2. --- README.md | 48 +++++++++++++++++++ .../cloud/gce/GceComputeService.java | 43 ++++++++++++++++- src/test/resources/elasticsearch.yml | 2 + 3 files changed, 91 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 7eb3218e8ab..a4585750b4d 100644 --- a/README.md +++ b/README.md @@ -243,6 +243,54 @@ gcutil --project=es-cloud deleteinstance myesnode1 myesnode2 --zone=europe-west1 gcutil --project=es-cloud deletedisk boot-myesnode1 boot-myesnode2 --zone=europe-west1-a ``` +Filtering by tags +----------------- + +The GCE discovery can also filter machines to include in the cluster based on tags using `discovery.gce.tags` settings. +For example, setting `discovery.gce.tags` to `dev` will only filter instances having a tag set to `dev`. Several tags +set will require all of those tags to be set for the instance to be included. + +One practical use for tag filtering is when an GCE cluster contains many nodes that are not running +elasticsearch. In this case (particularly with high ping_timeout values) there is a risk that a new node's discovery +phase will end before it has found the cluster (which will result in it declaring itself master of a new cluster +with the same name - highly undesirable). Adding tag on elasticsearch GCE nodes and then filtering by that +tag will resolve this issue. + +Add your tag when building the new instance: + +```sh +gcutil --project=es-cloud addinstance myesnode1 --service_account_scope=compute-rw --persistent_boot_disk \ + --tags=elasticsearch,dev +``` + +Then, define it in `elasticsearch.yml`: + +```yaml + cloud: + gce: + project_id: es-cloud + zone: europe-west1-a + discovery: + type: gce + gce: + tags: elasticsearch, dev +``` + +Tips +---- + +If you don't want to repeat the project id each time, you can save it in `~/.gcutil.flags` file using: + +```sh +gcutil getproject --project=es-cloud --cache_flag_values +``` + +`~/.gcutil.flags` file now contains: + +``` +--project=es-cloud +``` + License ------- diff --git a/src/main/java/org/elasticsearch/cloud/gce/GceComputeService.java b/src/main/java/org/elasticsearch/cloud/gce/GceComputeService.java index 8957384bd94..174dd283195 100644 --- a/src/main/java/org/elasticsearch/cloud/gce/GceComputeService.java +++ b/src/main/java/org/elasticsearch/cloud/gce/GceComputeService.java @@ -63,6 +63,8 @@ public class GceComputeService extends AbstractLifecycleComponent 0) { + if (instance.getTags() == null || instance.getTags().isEmpty()) { + // If this instance have no tag, we filter it + filterByTag = true; + } else { + // check that all tags listed are there on the instance + for (String tag : tags) { + boolean found = false; + for (String instancetag : instance.getTags().getItems()) { + if (instancetag.equals(tag)) { + found = true; + break; + } + } + if (!found) { + filterByTag = true; + break; + } + } + } + } + if (filterByTag) { + logger.trace("filtering out instance {} based tags {}, not part of {}", name, tags, + instance.getTags().getItems()); + continue; + } + String ip_public = null; String ip_private = null; @@ -133,12 +171,13 @@ public class GceComputeService extends AbstractLifecycleComponent Date: Fri, 28 Jun 2013 15:49:18 +0200 Subject: [PATCH 04/96] Ignore TERMINATED instances When an instance is removed, its status become `TERMINATED`. As stated in [GCE Documentation](https://developers.google.com/compute/docs/instances#checkmachinestatus): > TERMINATED - The instance either failed for some reason or was shutdown. This is a permanent status, and the only way to repair the instance is to delete and recreate it. So we need to ignore instances with such a status. Closes #3. --- .../elasticsearch/cloud/gce/GceComputeService.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/main/java/org/elasticsearch/cloud/gce/GceComputeService.java b/src/main/java/org/elasticsearch/cloud/gce/GceComputeService.java index 174dd283195..05be0ecdaf0 100644 --- a/src/main/java/org/elasticsearch/cloud/gce/GceComputeService.java +++ b/src/main/java/org/elasticsearch/cloud/gce/GceComputeService.java @@ -58,6 +58,10 @@ public class GceComputeService extends AbstractLifecycleComponent discoNodes; private TransportService transportService; private NetworkService networkService; @@ -114,6 +118,13 @@ public class GceComputeService extends AbstractLifecycleComponent 0) { From e3b08e348142eea1b0e88b641800895f37bd5f19 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Fri, 28 Jun 2013 16:25:19 +0200 Subject: [PATCH 05/96] Update documentation. Thanks to @spinscale --- README.md | 67 +++++++++++++++++++++---------------------------------- 1 file changed, 26 insertions(+), 41 deletions(-) diff --git a/README.md b/README.md index a4585750b4d..0466c67119e 100644 --- a/README.md +++ b/README.md @@ -107,11 +107,8 @@ sudo apt-get install java7-runtime-headless # Prepare Elasticsearch installation sudo dpkg -i elasticsearch-0.90.2.deb -``` -Check that elasticsearch is running: - -```sh +# Check that elasticsearch is running: curl http://localhost:9200/ ``` @@ -132,21 +129,13 @@ This command should give you a JSON result: ### Install elasticsearch cloud gce plugin -Stop elasticsearch: - -```sh -sudo service elasticsearch stop -``` - Install the plugin: ```sh +# Use Plugin Manager to install it sudo /usr/share/elasticsearch/bin/plugin -install elasticsearch/elasticsearch-cloud-gce/1.0.0-SNAPSHOT -``` -Configure it: - -```sh +# Configure it: sudo vi /etc/elasticsearch/elasticsearch.yml ``` @@ -161,54 +150,47 @@ And add the following lines: type: gce ``` + Restart elasticsearch: ```sh -sudo service elasticsearch start +sudo /etc/init.d/elasticsearch restart ``` ### Cloning your existing machine -First create an image of your running instance: +In order to build a cluster on many nodes, you can clone your configured instance to new nodes. +You won't have to reinstall everything! + +First create an image of your running instance and upload it to Google Cloud Storage: ```sh +# Create an image of yur current instance sudo python /usr/share/imagebundle/image_bundle.py \ -r / -o /tmp/ --log_file=/tmp/abc.log -``` -An image has been created in `/tmp` directory: - -```sh -$ ls /tmp +# An image has been created in `/tmp` directory: +ls /tmp e4686d7f5bf904a924ae0cfeb58d0827c6d5b966.image.tar.gz -``` -Upload your image to Google Cloud Storage: - -```sh +# Upload your image to Google Cloud Storage: # Launch this command and follow instructions to give your instance an access to your storage gsutil config -``` -Create a bucket to hold your image, let's say `esimage`: - -```sh +# Create a bucket to hold your image, let's say `esimage`: gsutil mb gs://esimage -``` -Copy your image to this bucket: - -```sh +# Copy your image to this bucket: gsutil cp /tmp/e4686d7f5bf904a924ae0cfeb58d0827c6d5b966.image.tar.gz gs://esimage + +# You can logout +exit ``` -Add your image to images collection: - -First, exit from your SSH session: `exit`. - -Then, run locally: +Then add your image to images collection: ```sh +# From your own computer (not from google compute engine instance!) gcutil listkernels --project es-cloud +----------------------------------------------+--------------------------------------------------+-------------+ | name | description | deprecation | @@ -216,22 +198,25 @@ gcutil listkernels --project es-cloud | projects/google/global/kernels/gce-20120621 | 2.6.39-gcg built 2012-03-29 01:07:00 | DEPRECATED | | projects/google/global/kernels/gce-v20130603 | SCSI-enabled 3.3.8-gcg built 2013-05-29 01:04:00 | | +----------------------------------------------+--------------------------------------------------+-------------+ -# Note the kernel you prefer to use +# Note the kernel you prefer to use and add your image to your catalog: gcutil --project=es-cloud addimage elasticsearch-0-90-2 \ gs://esimage/e4686d7f5bf904a924ae0cfeb58d0827c6d5b966.image.tar.gz \ --preferred_kernel=projects/google/global/kernels/gce-v20130603 ``` -Create as many instance as you need: +### Start new instances + +As you have now an image, you can create as many instance as you need: ```sh +# Just change node name (here myesnode2) gcutil --project=es-cloud addinstance --image=elasticsearch-0-90-2 \ --kernel=projects/google/global/kernels/gce-v20130603 myesnode2 \ --zone europe-west1-a --machine_type f1-micro --service_account_scope=compute-rw \ --persistent_boot_disk ``` -### Remove a node +### Remove an instance (aka shut it down) You can use [Google Cloud Console](https://cloud.google.com/console) or CLI to manage your instances: From e734028ae21ea9ac3e8ac5ae0f4c61f57ce45ebc Mon Sep 17 00:00:00 2001 From: David Pilato Date: Fri, 28 Jun 2013 16:27:13 +0200 Subject: [PATCH 06/96] Google service account name is not required --- README.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/README.md b/README.md index 0466c67119e..31be4d6c0df 100644 --- a/README.md +++ b/README.md @@ -44,10 +44,8 @@ How to start (long story) Before starting, you should have: -* [GCUtil](https://developers.google.com/compute/docs/gcutil/#install) -* Your Google service account name. Generally, the account name has the format `123845678986@project.gserviceaccount.com`. -You can get it from [Google APIS Console](https://code.google.com/apis/console/) * Your project ID. Let's say here `es-cloud`. Get it from [Google APIS Console](https://code.google.com/apis/console/). +* [GCUtil](https://developers.google.com/compute/docs/gcutil/#install) ### Creating your first instance From db86c13a4e1e56b24c69157ce40a4a58109fbdfa Mon Sep 17 00:00:00 2001 From: David Pilato Date: Fri, 28 Jun 2013 16:29:40 +0200 Subject: [PATCH 07/96] Update documentation. Thanks to @spinscale --- README.md | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 31be4d6c0df..147a0f09230 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -Google Compute Engine Cloud Plugin for ElasticSearch +Google Compute Engine Cloud Plugin for Elasticsearch ==================================================== The GCE Cloud plugin allows to use GCE API for the unicast discovery mechanism. @@ -6,7 +6,7 @@ The GCE Cloud plugin allows to use GCE API for the unicast discovery mechanism. In order to install the plugin, simply run: `bin/plugin -install elasticsearch/elasticsearch-cloud-gce/1.0.0-SNAPSHOT`. ---------------------------------------------- - | GCE Cloud Plugin | ElasticSearch | + | GCE Cloud Plugin | Elasticsearch | ---------------------------------------------- | 1.0.0-SNAPSHOT (master) | 0.90 -> master | ---------------------------------------------- @@ -80,10 +80,7 @@ Table of resources: +-----------+--------------+-------+---------+--------------+----------------+----------------+----------------+---------+----------------+ ``` -Note your external ip address. - -Now, your first instance is started. You need to install Elasticsearch on it. -You can now connect to your machine: +You can now connect to your machine using the external IP address in order to install Elasticsearch: ``` ssh -i ~/.ssh/google_compute_engine 192.158.29.199 @@ -204,7 +201,7 @@ gcutil --project=es-cloud addimage elasticsearch-0-90-2 \ ### Start new instances -As you have now an image, you can create as many instance as you need: +As you have now an image, you can create as many instances as you need: ```sh # Just change node name (here myesnode2) From 1602d82186774610fdec1ee4c008ddc3489a7395 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Mon, 1 Jul 2013 17:56:41 +0200 Subject: [PATCH 08/96] Move all discovery process to GceUnicastHostsProvider We want to be able to rebuild the full list of existing nodes each time we need it so we don't want to have a fixed list of nodes. --- README.md | 11 +- .../cloud/gce/GceComputeService.java | 208 ++---------------- .../discovery/gce/GceDiscovery.java | 7 +- .../gce/GceUnicastHostsProvider.java | 195 +++++++++++++++- 4 files changed, 219 insertions(+), 202 deletions(-) diff --git a/README.md b/README.md index 147a0f09230..49fd6253797 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ Before starting, you should have: ```sh -gcutil --project=es-cloud addinstance myesnode1 --service_account_scope=compute-rw --persistent_boot_disk +gcutil --project=es-cloud addinstance myesnode1 --service_account_scope=compute-rw,storage-full --persistent_boot_disk ``` You will be asked to open a link in your browser. Login and allow access to listed services. @@ -178,14 +178,7 @@ gsutil mb gs://esimage # Copy your image to this bucket: gsutil cp /tmp/e4686d7f5bf904a924ae0cfeb58d0827c6d5b966.image.tar.gz gs://esimage -# You can logout -exit -``` - -Then add your image to images collection: - -```sh -# From your own computer (not from google compute engine instance!) +# Then add your image to images collection: gcutil listkernels --project es-cloud +----------------------------------------------+--------------------------------------------------+-------------+ | name | description | deprecation | diff --git a/src/main/java/org/elasticsearch/cloud/gce/GceComputeService.java b/src/main/java/org/elasticsearch/cloud/gce/GceComputeService.java index 05be0ecdaf0..6f76fd82b02 100644 --- a/src/main/java/org/elasticsearch/cloud/gce/GceComputeService.java +++ b/src/main/java/org/elasticsearch/cloud/gce/GceComputeService.java @@ -25,27 +25,13 @@ import com.google.api.client.http.HttpTransport; import com.google.api.client.json.JsonFactory; import com.google.api.client.json.jackson2.JacksonFactory; import com.google.api.services.compute.Compute; -import com.google.api.services.compute.model.AccessConfig; -import com.google.api.services.compute.model.Instance; -import com.google.api.services.compute.model.InstanceList; -import com.google.api.services.compute.model.NetworkInterface; import org.elasticsearch.ElasticSearchException; -import org.elasticsearch.cluster.node.DiscoveryNode; -import org.elasticsearch.common.Strings; -import org.elasticsearch.common.collect.Lists; import org.elasticsearch.common.component.AbstractLifecycleComponent; import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.network.NetworkService; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; -import org.elasticsearch.common.transport.TransportAddress; -import org.elasticsearch.discovery.zen.ping.unicast.UnicastZenPing; -import org.elasticsearch.transport.TransportService; - -import java.io.IOException; -import java.net.InetAddress; -import java.security.GeneralSecurityException; -import java.util.List; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.discovery.DiscoveryException; /** * @@ -53,21 +39,12 @@ import java.util.List; public class GceComputeService extends AbstractLifecycleComponent { static final class Fields { - private static final String PROJECT = "project_id"; - private static final String ZONE = "zone"; private static final String VERSION = "Elasticsearch/GceCloud/1.0"; } - static final class Status { - private static final String TERMINATED = "TERMINATED"; - } - - private List discoNodes; - private TransportService transportService; - private NetworkService networkService; - private Compute compute; - - private final String[] tags; + private Compute client; + private TimeValue refreshInterval = null; + private long lastRefresh; /** Global instance of the HTTP transport. */ private static HttpTransport HTTP_TRANSPORT; @@ -76,178 +53,43 @@ public class GceComputeService extends AbstractLifecycleComponent buildNodes(String project, String zone) throws IOException { - - List discoNodes = Lists.newArrayList(); - String ipAddress = null; - try { - InetAddress inetAddress = networkService.resolvePublishHostAddress(null); - if (inetAddress != null) { - ipAddress = inetAddress.getHostAddress(); + public synchronized Compute client() { + if (refreshInterval != null && refreshInterval.millis() != 0) { + if (client != null && + (refreshInterval.millis() < 0 || (System.currentTimeMillis() - lastRefresh) < refreshInterval.millis())) { + if (logger.isTraceEnabled()) logger.trace("using cache to retrieve client"); + return client; } - } catch (IOException e) { - // We can't find the publish host address... Hmmm. Too bad :-( + lastRefresh = System.currentTimeMillis(); } - Compute.Instances.List list = compute.instances().list(project, zone); - - InstanceList instanceList = list.execute(); - - for (Instance instance : instanceList.getItems()) { - String name = instance.getName(); - String type = instance.getMachineType(); - String image = instance.getImage(); - - String status = instance.getStatus(); - - // We don't want to connect to TERMINATED status instances - // See https://github.com/elasticsearch/elasticsearch-cloud-gce/issues/3 - if (Status.TERMINATED.equals(status)) { - logger.debug("node {} is TERMINATED. Ignoring", name); - continue; - } - - // see if we need to filter by tag - boolean filterByTag = false; - if (tags.length > 0) { - if (instance.getTags() == null || instance.getTags().isEmpty()) { - // If this instance have no tag, we filter it - filterByTag = true; - } else { - // check that all tags listed are there on the instance - for (String tag : tags) { - boolean found = false; - for (String instancetag : instance.getTags().getItems()) { - if (instancetag.equals(tag)) { - found = true; - break; - } - } - if (!found) { - filterByTag = true; - break; - } - } - } - } - if (filterByTag) { - logger.trace("filtering out instance {} based tags {}, not part of {}", name, tags, - instance.getTags().getItems()); - continue; - } - - String ip_public = null; - String ip_private = null; - - List interfaces = instance.getNetworkInterfaces(); - - for (NetworkInterface networkInterface : interfaces) { - if (ip_public == null) { - // Trying to get Public IP Address (For future use) - for (AccessConfig accessConfig : networkInterface.getAccessConfigs()) { - if (Strings.hasText(accessConfig.getNatIP())) { - ip_public = accessConfig.getNatIP(); - break; - } - } - } - - if (ip_private == null) { - ip_private = networkInterface.getNetworkIP(); - } - - // If we have both public and private, we can stop here - if (ip_private != null && ip_public != null) break; - } - - try { - if (ip_private.equals(ipAddress)) { - // We found the current node. - // We can ignore it in the list of DiscoveryNode - logger.debug("current node found. Ignoring {} - {}", name, ip_private); - } else { - TransportAddress[] addresses = transportService.addressesFromString(ip_private); - // we only limit to 1 addresses, makes no sense to ping 100 ports - for (int i = 0; (i < addresses.length && i < UnicastZenPing.LIMIT_PORTS_COUNT); i++) { - logger.trace("adding {}, type {}, image {}, address {}, transport_address {}, status {}", name, type, - image, ip_private, addresses[i], status); - discoNodes.add(new DiscoveryNode("#cloud-" + name + "-" + i, addresses[i])); - } - } - } catch (Exception e) { - logger.warn("failed to add {}, address {}", e, name, ip_private); - } - - } - - - return discoNodes; - } - - public synchronized List nodes() { - if (this.discoNodes != null) { - return this.discoNodes; - } try { HTTP_TRANSPORT = GoogleNetHttpTransport.newTrustedTransport(); - } catch (GeneralSecurityException e) { + JSON_FACTORY = new JacksonFactory(); - } catch (IOException e) { - - } - JSON_FACTORY = new JacksonFactory(); - - - String project = componentSettings.get(Fields.PROJECT, settings.get("cloud." + Fields.PROJECT)); - String zone = componentSettings.get(Fields.ZONE, settings.get("cloud." + Fields.ZONE)); - - // Check that we have all needed properties - if (!checkProperty(Fields.PROJECT, project)) return null; - if (!checkProperty(Fields.ZONE, zone)) return null; - - try { - logger.debug("starting GCE discovery service for project [{}] on zone [{}]", project, zone); + logger.debug("starting GCE discovery service"); ComputeCredential credential = new ComputeCredential.Builder(HTTP_TRANSPORT, JSON_FACTORY).build(); credential.refreshToken(); logger.debug("token [{}] will expire in [{}] s", credential.getAccessToken(), credential.getExpiresInSeconds()); + refreshInterval = TimeValue.timeValueSeconds(credential.getExpiresInSeconds()-1); // Once done, let's use this token - compute = new Compute.Builder(HTTP_TRANSPORT, JSON_FACTORY, null) + this.client = new Compute.Builder(HTTP_TRANSPORT, JSON_FACTORY, null) .setApplicationName(Fields.VERSION) .setHttpRequestInitializer(credential) .build(); - - this.discoNodes = buildNodes(project, zone); - } catch (Throwable t) { - logger.warn("error while trying to find nodes for GCE service [{}]: {}: {}", project, t.getClass().getName(), - t.getMessage()); - logger.debug("error found is: ", t); - // We create an empty list in that specific case. - // So discovery process won't fail with NPE but this node won't join any cluster - this.discoNodes = Lists.newArrayList(); + } catch (Exception e) { + logger.warn("unable to start GCE discovery service: {} : {}", e.getClass().getName(), e.getMessage()); + throw new DiscoveryException("unable to start GCE discovery service", e); } - logger.debug("using dynamic discovery nodes {}", discoNodes); - return this.discoNodes; + return this.client; } @Override @@ -261,12 +103,4 @@ public class GceComputeService extends AbstractLifecycleComponent zenPings = pingService.zenPings(); @@ -57,7 +59,8 @@ public class GceDiscovery extends ZenDiscovery { if (unicastZenPing != null) { // update the unicast zen ping to add cloud hosts provider // and, while we are at it, use only it and not the multicast for example - unicastZenPing.addHostsProvider(new GceUnicastHostsProvider(settings, gceComputeService.nodes())); + unicastZenPing.addHostsProvider(new GceUnicastHostsProvider(settings, gceComputeService, + transportService, networkService)); pingService.zenPings(ImmutableList.of(unicastZenPing)); } else { logger.warn("failed to apply gce unicast discovery, no unicast ping found"); diff --git a/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java b/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java index 358a9860c58..834c1c0f9c2 100644 --- a/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java +++ b/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java @@ -19,12 +19,27 @@ package org.elasticsearch.discovery.gce; +import com.google.api.services.compute.Compute; +import com.google.api.services.compute.model.AccessConfig; +import com.google.api.services.compute.model.Instance; +import com.google.api.services.compute.model.InstanceList; +import com.google.api.services.compute.model.NetworkInterface; +import org.elasticsearch.cloud.gce.GceComputeService; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.collect.Lists; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.network.NetworkService; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.discovery.zen.ping.unicast.UnicastHostsProvider; +import org.elasticsearch.discovery.zen.ping.unicast.UnicastZenPing; +import org.elasticsearch.transport.TransportService; +import java.io.IOException; +import java.net.InetAddress; import java.util.List; /** @@ -32,16 +47,188 @@ import java.util.List; */ public class GceUnicastHostsProvider extends AbstractComponent implements UnicastHostsProvider { - private final List discoNodes; + static final class Fields { + private static final String PROJECT = "project_id"; + private static final String ZONE = "zone"; + private static final String REFRESH = "refresh_interval"; + } + + static final class Status { + private static final String TERMINATED = "TERMINATED"; + } + + private final GceComputeService gceComputeService; + private TransportService transportService; + private NetworkService networkService; + + private final String project; + private final String zone; + private final String[] tags; + + private final TimeValue refreshInterval; + private long lastRefresh; + private List cachedDiscoNodes; @Inject - public GceUnicastHostsProvider(Settings settings, List discoNodes) { + public GceUnicastHostsProvider(Settings settings, GceComputeService gceComputeService, + TransportService transportService, + NetworkService networkService) { super(settings); - this.discoNodes = discoNodes; + this.gceComputeService = gceComputeService; + this.transportService = transportService; + this.networkService = networkService; + + this.refreshInterval = componentSettings.getAsTime(Fields.REFRESH, + settings.getAsTime("cloud.gce." + Fields.REFRESH, TimeValue.timeValueSeconds(5))); + + this.project = componentSettings.get(Fields.PROJECT, settings.get("cloud.gce." + Fields.PROJECT)); + this.zone = componentSettings.get(Fields.ZONE, settings.get("cloud.gce." + Fields.ZONE)); + + // Check that we have all needed properties + checkProperty(Fields.PROJECT, project); + checkProperty(Fields.ZONE, zone); + + this.tags = settings.getAsArray("discovery.gce.tags"); + if (logger.isDebugEnabled()) { + logger.debug("using tags {}", Lists.newArrayList(this.tags)); + } } + /** + * We build the list of Nodes from GCE Management API + * Information are cached for 5 seconds by default. Modify `plugins.refresh_interval` property if needed. + * Setting `plugins.refresh_interval` to `-1` will cause infinite caching. + * Setting `plugins.refresh_interval` to `0` will disable caching. + */ @Override public List buildDynamicNodes() { - return discoNodes; + if (refreshInterval.millis() != 0) { + if (cachedDiscoNodes != null && + (refreshInterval.millis() < 0 || (System.currentTimeMillis() - lastRefresh) < refreshInterval.millis())) { + if (logger.isTraceEnabled()) logger.trace("using cache to retrieve node list"); + return cachedDiscoNodes; + } + lastRefresh = System.currentTimeMillis(); + } + logger.debug("start building nodes list using GCE API"); + + cachedDiscoNodes = Lists.newArrayList(); + String ipAddress = null; + try { + InetAddress inetAddress = networkService.resolvePublishHostAddress(null); + if (inetAddress != null) { + ipAddress = inetAddress.getHostAddress(); + } + } catch (IOException e) { + // We can't find the publish host address... Hmmm. Too bad :-( + // We won't simply filter it + } + + try { + Compute.Instances.List list = gceComputeService.client().instances().list(project, zone); + + InstanceList instanceList = list.execute(); + + for (Instance instance : instanceList.getItems()) { + String name = instance.getName(); + String type = instance.getMachineType(); + String image = instance.getImage(); + + String status = instance.getStatus(); + + // We don't want to connect to TERMINATED status instances + // See https://github.com/elasticsearch/elasticsearch-cloud-gce/issues/3 + if (Status.TERMINATED.equals(status)) { + logger.debug("node {} is TERMINATED. Ignoring", name); + continue; + } + + // see if we need to filter by tag + boolean filterByTag = false; + if (tags.length > 0) { + if (instance.getTags() == null || instance.getTags().isEmpty()) { + // If this instance have no tag, we filter it + filterByTag = true; + } else { + // check that all tags listed are there on the instance + for (String tag : tags) { + boolean found = false; + for (String instancetag : instance.getTags().getItems()) { + if (instancetag.equals(tag)) { + found = true; + break; + } + } + if (!found) { + filterByTag = true; + break; + } + } + } + } + if (filterByTag) { + logger.trace("filtering out instance {} based tags {}, not part of {}", name, tags, + instance.getTags().getItems()); + continue; + } + + String ip_public = null; + String ip_private = null; + + List interfaces = instance.getNetworkInterfaces(); + + for (NetworkInterface networkInterface : interfaces) { + if (ip_public == null) { + // Trying to get Public IP Address (For future use) + for (AccessConfig accessConfig : networkInterface.getAccessConfigs()) { + if (Strings.hasText(accessConfig.getNatIP())) { + ip_public = accessConfig.getNatIP(); + break; + } + } + } + + if (ip_private == null) { + ip_private = networkInterface.getNetworkIP(); + } + + // If we have both public and private, we can stop here + if (ip_private != null && ip_public != null) break; + } + + try { + if (ip_private.equals(ipAddress)) { + // We found the current node. + // We can ignore it in the list of DiscoveryNode + logger.trace("current node found. Ignoring {} - {}", name, ip_private); + } else { + TransportAddress[] addresses = transportService.addressesFromString(ip_private); + // we only limit to 1 addresses, makes no sense to ping 100 ports + for (int i = 0; (i < addresses.length && i < UnicastZenPing.LIMIT_PORTS_COUNT); i++) { + logger.trace("adding {}, type {}, image {}, address {}, transport_address {}, status {}", name, type, + image, ip_private, addresses[i], status); + cachedDiscoNodes.add(new DiscoveryNode("#cloud-" + name + "-" + i, addresses[i])); + } + } + } catch (Exception e) { + logger.warn("failed to add {}, address {}", e, name, ip_private); + } + + } + } catch (IOException e) { + logger.warn("Exception caught during discovery {} : {}", e.getClass().getName(), e.getMessage()); + logger.trace("Exception caught during discovery", e); + } + + logger.debug("using dynamic discovery nodes {}", cachedDiscoNodes); + return cachedDiscoNodes; + } + + private boolean checkProperty(String name, String value) { + if (!Strings.hasText(value)) { + logger.warn("cloud.gce.{} is not set.", name); + return false; + } + return true; } } From 530ae2169e7027e8ff08e4d940d83c1ac089d811 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Mon, 1 Jul 2013 18:13:05 +0200 Subject: [PATCH 09/96] By default, we don't cache GCE API calls `refresh_interval` = 0. --- .../discovery/gce/GceUnicastHostsProvider.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java b/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java index 834c1c0f9c2..76ec9831acb 100644 --- a/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java +++ b/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java @@ -79,7 +79,7 @@ public class GceUnicastHostsProvider extends AbstractComponent implements Unicas this.networkService = networkService; this.refreshInterval = componentSettings.getAsTime(Fields.REFRESH, - settings.getAsTime("cloud.gce." + Fields.REFRESH, TimeValue.timeValueSeconds(5))); + settings.getAsTime("cloud.gce." + Fields.REFRESH, TimeValue.timeValueSeconds(0))); this.project = componentSettings.get(Fields.PROJECT, settings.get("cloud.gce." + Fields.PROJECT)); this.zone = componentSettings.get(Fields.ZONE, settings.get("cloud.gce." + Fields.ZONE)); @@ -96,9 +96,9 @@ public class GceUnicastHostsProvider extends AbstractComponent implements Unicas /** * We build the list of Nodes from GCE Management API - * Information are cached for 5 seconds by default. Modify `plugins.refresh_interval` property if needed. + * Information can be cached using `plugins.refresh_interval` property if needed. * Setting `plugins.refresh_interval` to `-1` will cause infinite caching. - * Setting `plugins.refresh_interval` to `0` will disable caching. + * Setting `plugins.refresh_interval` to `0` will disable caching (default). */ @Override public List buildDynamicNodes() { From 0d40a0e2f9de27251308261f147031e49481d78a Mon Sep 17 00:00:00 2001 From: David Pilato Date: Mon, 1 Jul 2013 23:17:14 +0200 Subject: [PATCH 10/96] Trying to Mock GCE API for tests --- pom.xml | 27 ++++ .../cloud/gce/GceComputeService.java | 88 ++---------- .../cloud/gce/GceComputeServiceImpl.java | 127 ++++++++++++++++++ .../elasticsearch/cloud/gce/GceModule.java | 2 +- .../gce/GceUnicastHostsProvider.java | 11 +- .../plugin/cloud/gce/CloudGcePlugin.java | 4 +- .../gce/tests/GceComputeServiceMock.java | 60 +++++++++ 7 files changed, 231 insertions(+), 88 deletions(-) create mode 100644 src/main/java/org/elasticsearch/cloud/gce/GceComputeServiceImpl.java create mode 100644 src/test/java/org/elasticsearch/cloud/gce/tests/GceComputeServiceMock.java diff --git a/pom.xml b/pom.xml index 4b8c8de3c3a..bd83e6a476b 100644 --- a/pom.xml +++ b/pom.xml @@ -99,6 +99,14 @@ governing permissions and limitations under the License. --> + + + org.kuali.maven.wagons + maven-s3-wagon + 1.1.19 + + + src/main/resources @@ -153,6 +161,25 @@ governing permissions and limitations under the License. --> + + org.codehaus.mojo + wagon-maven-plugin + 1.0-beta-4 + + elasticsearch.download + ${project.build.directory}/releases/${project.build.finalName}.zip + s3://test.elasticsearch.org/elasticsearch/${project.artifactId}/ + + + + upload-zip-to-server + deploy + + upload-single + + + + diff --git a/src/main/java/org/elasticsearch/cloud/gce/GceComputeService.java b/src/main/java/org/elasticsearch/cloud/gce/GceComputeService.java index 6f76fd82b02..262aa8834c2 100644 --- a/src/main/java/org/elasticsearch/cloud/gce/GceComputeService.java +++ b/src/main/java/org/elasticsearch/cloud/gce/GceComputeService.java @@ -19,88 +19,20 @@ package org.elasticsearch.cloud.gce; -import com.google.api.client.googleapis.compute.ComputeCredential; -import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; -import com.google.api.client.http.HttpTransport; -import com.google.api.client.json.JsonFactory; -import com.google.api.client.json.jackson2.JacksonFactory; -import com.google.api.services.compute.Compute; -import org.elasticsearch.ElasticSearchException; -import org.elasticsearch.common.component.AbstractLifecycleComponent; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.settings.SettingsFilter; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.discovery.DiscoveryException; +import com.google.api.services.compute.model.Instance; + +import java.util.Collection; /** * */ -public class GceComputeService extends AbstractLifecycleComponent { - - static final class Fields { - private static final String VERSION = "Elasticsearch/GceCloud/1.0"; +public interface GceComputeService { + static final public class Fields { + public static final String PROJECT = "project_id"; + public static final String ZONE = "zone"; + public static final String REFRESH = "refresh_interval"; + public static final String VERSION = "Elasticsearch/GceCloud/1.0"; } - private Compute client; - private TimeValue refreshInterval = null; - private long lastRefresh; - - /** Global instance of the HTTP transport. */ - private static HttpTransport HTTP_TRANSPORT; - - /** Global instance of the JSON factory. */ - private static JsonFactory JSON_FACTORY; - - @Inject - public GceComputeService(Settings settings, SettingsFilter settingsFilter) { - super(settings); - settingsFilter.addFilter(new GceSettingsFilter()); - } - - public synchronized Compute client() { - if (refreshInterval != null && refreshInterval.millis() != 0) { - if (client != null && - (refreshInterval.millis() < 0 || (System.currentTimeMillis() - lastRefresh) < refreshInterval.millis())) { - if (logger.isTraceEnabled()) logger.trace("using cache to retrieve client"); - return client; - } - lastRefresh = System.currentTimeMillis(); - } - - try { - HTTP_TRANSPORT = GoogleNetHttpTransport.newTrustedTransport(); - JSON_FACTORY = new JacksonFactory(); - - logger.debug("starting GCE discovery service"); - ComputeCredential credential = new ComputeCredential.Builder(HTTP_TRANSPORT, JSON_FACTORY).build(); - credential.refreshToken(); - - logger.debug("token [{}] will expire in [{}] s", credential.getAccessToken(), credential.getExpiresInSeconds()); - refreshInterval = TimeValue.timeValueSeconds(credential.getExpiresInSeconds()-1); - - // Once done, let's use this token - this.client = new Compute.Builder(HTTP_TRANSPORT, JSON_FACTORY, null) - .setApplicationName(Fields.VERSION) - .setHttpRequestInitializer(credential) - .build(); - } catch (Exception e) { - logger.warn("unable to start GCE discovery service: {} : {}", e.getClass().getName(), e.getMessage()); - throw new DiscoveryException("unable to start GCE discovery service", e); - } - - return this.client; - } - - @Override - protected void doStart() throws ElasticSearchException { - } - - @Override - protected void doStop() throws ElasticSearchException { - } - - @Override - protected void doClose() throws ElasticSearchException { - } + public Collection instances(); } diff --git a/src/main/java/org/elasticsearch/cloud/gce/GceComputeServiceImpl.java b/src/main/java/org/elasticsearch/cloud/gce/GceComputeServiceImpl.java new file mode 100644 index 00000000000..0dd3f2da3db --- /dev/null +++ b/src/main/java/org/elasticsearch/cloud/gce/GceComputeServiceImpl.java @@ -0,0 +1,127 @@ +/* + * Licensed to ElasticSearch under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.cloud.gce; + +import com.google.api.client.googleapis.compute.ComputeCredential; +import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; +import com.google.api.client.http.HttpTransport; +import com.google.api.client.json.JsonFactory; +import com.google.api.client.json.jackson2.JacksonFactory; +import com.google.api.services.compute.Compute; +import com.google.api.services.compute.model.Instance; +import com.google.api.services.compute.model.InstanceList; +import org.elasticsearch.ElasticSearchException; +import org.elasticsearch.common.component.AbstractLifecycleComponent; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.settings.SettingsFilter; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.discovery.DiscoveryException; + +import java.io.IOException; +import java.util.Collection; + +/** + * + */ +public class GceComputeServiceImpl extends AbstractLifecycleComponent + implements GceComputeService { + + private final String project; + private final String zone; + + @Override + public Collection instances() { + try { + Compute.Instances.List list = client().instances().list(project, zone); + InstanceList instanceList = list.execute(); + + return instanceList.getItems(); + } catch (IOException e) { + throw new ElasticSearchException("Pas glop"); + } + + } + + private Compute client; + private TimeValue refreshInterval = null; + private long lastRefresh; + + /** Global instance of the HTTP transport. */ + private static HttpTransport HTTP_TRANSPORT; + + /** Global instance of the JSON factory. */ + private static JsonFactory JSON_FACTORY; + + @Inject + public GceComputeServiceImpl(Settings settings, SettingsFilter settingsFilter) { + super(settings); + settingsFilter.addFilter(new GceSettingsFilter()); + + this.project = componentSettings.get(Fields.PROJECT, settings.get("cloud.gce." + Fields.PROJECT)); + this.zone = componentSettings.get(Fields.ZONE, settings.get("cloud.gce." + Fields.ZONE)); + } + + public synchronized Compute client() { + if (refreshInterval != null && refreshInterval.millis() != 0) { + if (client != null && + (refreshInterval.millis() < 0 || (System.currentTimeMillis() - lastRefresh) < refreshInterval.millis())) { + if (logger.isTraceEnabled()) logger.trace("using cache to retrieve client"); + return client; + } + lastRefresh = System.currentTimeMillis(); + } + + try { + HTTP_TRANSPORT = GoogleNetHttpTransport.newTrustedTransport(); + JSON_FACTORY = new JacksonFactory(); + + logger.debug("starting GCE discovery service"); + ComputeCredential credential = new ComputeCredential.Builder(HTTP_TRANSPORT, JSON_FACTORY).build(); + credential.refreshToken(); + + logger.debug("token [{}] will expire in [{}] s", credential.getAccessToken(), credential.getExpiresInSeconds()); + refreshInterval = TimeValue.timeValueSeconds(credential.getExpiresInSeconds()-1); + + // Once done, let's use this token + this.client = new Compute.Builder(HTTP_TRANSPORT, JSON_FACTORY, null) + .setApplicationName(Fields.VERSION) + .setHttpRequestInitializer(credential) + .build(); + } catch (Exception e) { + logger.warn("unable to start GCE discovery service: {} : {}", e.getClass().getName(), e.getMessage()); + throw new DiscoveryException("unable to start GCE discovery service", e); + } + + return this.client; + } + + @Override + protected void doStart() throws ElasticSearchException { + } + + @Override + protected void doStop() throws ElasticSearchException { + } + + @Override + protected void doClose() throws ElasticSearchException { + } +} diff --git a/src/main/java/org/elasticsearch/cloud/gce/GceModule.java b/src/main/java/org/elasticsearch/cloud/gce/GceModule.java index 938c603bbf7..b6628dc5653 100644 --- a/src/main/java/org/elasticsearch/cloud/gce/GceModule.java +++ b/src/main/java/org/elasticsearch/cloud/gce/GceModule.java @@ -28,6 +28,6 @@ public class GceModule extends AbstractModule { @Override protected void configure() { - bind(GceComputeService.class).asEagerSingleton(); + bind(GceComputeService.class).to(GceComputeServiceImpl.class).asEagerSingleton(); } } diff --git a/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java b/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java index 76ec9831acb..98fbb80960f 100644 --- a/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java +++ b/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java @@ -19,10 +19,8 @@ package org.elasticsearch.discovery.gce; -import com.google.api.services.compute.Compute; import com.google.api.services.compute.model.AccessConfig; import com.google.api.services.compute.model.Instance; -import com.google.api.services.compute.model.InstanceList; import com.google.api.services.compute.model.NetworkInterface; import org.elasticsearch.cloud.gce.GceComputeService; import org.elasticsearch.cluster.node.DiscoveryNode; @@ -40,6 +38,7 @@ import org.elasticsearch.transport.TransportService; import java.io.IOException; import java.net.InetAddress; +import java.util.Collection; import java.util.List; /** @@ -125,11 +124,9 @@ public class GceUnicastHostsProvider extends AbstractComponent implements Unicas } try { - Compute.Instances.List list = gceComputeService.client().instances().list(project, zone); + Collection instances = gceComputeService.instances(); - InstanceList instanceList = list.execute(); - - for (Instance instance : instanceList.getItems()) { + for (Instance instance : instances) { String name = instance.getName(); String type = instance.getMachineType(); String image = instance.getImage(); @@ -215,7 +212,7 @@ public class GceUnicastHostsProvider extends AbstractComponent implements Unicas } } - } catch (IOException e) { + } catch (Throwable e) { logger.warn("Exception caught during discovery {} : {}", e.getClass().getName(), e.getMessage()); logger.trace("Exception caught during discovery", e); } diff --git a/src/main/java/org/elasticsearch/plugin/cloud/gce/CloudGcePlugin.java b/src/main/java/org/elasticsearch/plugin/cloud/gce/CloudGcePlugin.java index 0100a32c6eb..40be7338a09 100644 --- a/src/main/java/org/elasticsearch/plugin/cloud/gce/CloudGcePlugin.java +++ b/src/main/java/org/elasticsearch/plugin/cloud/gce/CloudGcePlugin.java @@ -19,7 +19,7 @@ package org.elasticsearch.plugin.cloud.gce; -import org.elasticsearch.cloud.gce.GceComputeService; +import org.elasticsearch.cloud.gce.GceComputeServiceImpl; import org.elasticsearch.cloud.gce.GceModule; import org.elasticsearch.common.collect.Lists; import org.elasticsearch.common.component.LifecycleComponent; @@ -63,7 +63,7 @@ public class CloudGcePlugin extends AbstractPlugin { public Collection> services() { Collection> services = Lists.newArrayList(); if (settings.getAsBoolean("cloud.enabled", true)) { - services.add(GceComputeService.class); + services.add(GceComputeServiceImpl.class); } return services; } diff --git a/src/test/java/org/elasticsearch/cloud/gce/tests/GceComputeServiceMock.java b/src/test/java/org/elasticsearch/cloud/gce/tests/GceComputeServiceMock.java new file mode 100644 index 00000000000..9d44d25e63b --- /dev/null +++ b/src/test/java/org/elasticsearch/cloud/gce/tests/GceComputeServiceMock.java @@ -0,0 +1,60 @@ +/* + * Licensed to ElasticSearch under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.cloud.gce.tests; + +import com.google.api.services.compute.model.Instance; +import org.elasticsearch.ElasticSearchException; +import org.elasticsearch.cloud.gce.GceComputeService; +import org.elasticsearch.common.component.AbstractLifecycleComponent; +import org.elasticsearch.common.settings.Settings; + +import java.util.ArrayList; +import java.util.Collection; + +/** + * + */ +public class GceComputeServiceMock extends AbstractLifecycleComponent + implements GceComputeService { + + protected GceComputeServiceMock(Settings settings) { + super(settings); + logger.debug("Starting MOCK"); + } + + @Override + public Collection instances() { + Collection instances = new ArrayList(); + + return instances; + } + + @Override + protected void doStart() throws ElasticSearchException { + } + + @Override + protected void doStop() throws ElasticSearchException { + } + + @Override + protected void doClose() throws ElasticSearchException { + } +} From ca45569f0ee91abcfe522eb7e4b6c52b8577cd17 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Mon, 8 Jul 2013 15:15:41 +0200 Subject: [PATCH 11/96] Adding JUnit test using GCE API Mock --- .../cloud/gce/GceComputeServiceImpl.java | 5 +- .../elasticsearch/cloud/gce/GceModule.java | 10 +- .../gce/GceUnicastHostsProvider.java | 20 +++- .../plugin/cloud/gce/CloudGcePlugin.java | 3 +- .../cloud/gce/tests/GceAbstractTest.java | 105 ++++++++++++++++++ ...ava => GceComputeServiceAbstractMock.java} | 34 +++++- ...mputeServiceTwoNodesDifferentTagsMock.java | 46 ++++++++ ...GceComputeServiceTwoNodesSameTagsMock.java | 48 ++++++++ .../cloud/gce/tests/GceDifferentTagsTest.java | 72 ++++++++++++ .../cloud/gce/tests/GceSameTagsTest.java | 64 +++++++++++ .../cloud/gce/tests/GceSimpleTest.java | 60 ---------- src/test/resources/elasticsearch.yml | 4 +- 12 files changed, 396 insertions(+), 75 deletions(-) create mode 100644 src/test/java/org/elasticsearch/cloud/gce/tests/GceAbstractTest.java rename src/test/java/org/elasticsearch/cloud/gce/tests/{GceComputeServiceMock.java => GceComputeServiceAbstractMock.java} (54%) create mode 100644 src/test/java/org/elasticsearch/cloud/gce/tests/GceComputeServiceTwoNodesDifferentTagsMock.java create mode 100644 src/test/java/org/elasticsearch/cloud/gce/tests/GceComputeServiceTwoNodesSameTagsMock.java create mode 100644 src/test/java/org/elasticsearch/cloud/gce/tests/GceDifferentTagsTest.java create mode 100644 src/test/java/org/elasticsearch/cloud/gce/tests/GceSameTagsTest.java delete mode 100644 src/test/java/org/elasticsearch/cloud/gce/tests/GceSimpleTest.java diff --git a/src/main/java/org/elasticsearch/cloud/gce/GceComputeServiceImpl.java b/src/main/java/org/elasticsearch/cloud/gce/GceComputeServiceImpl.java index 0dd3f2da3db..38290bafe95 100644 --- a/src/main/java/org/elasticsearch/cloud/gce/GceComputeServiceImpl.java +++ b/src/main/java/org/elasticsearch/cloud/gce/GceComputeServiceImpl.java @@ -36,6 +36,7 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.discovery.DiscoveryException; import java.io.IOException; +import java.util.ArrayList; import java.util.Collection; /** @@ -55,9 +56,9 @@ public class GceComputeServiceImpl extends AbstractLifecycleComponent(); } - } private Compute client; diff --git a/src/main/java/org/elasticsearch/cloud/gce/GceModule.java b/src/main/java/org/elasticsearch/cloud/gce/GceModule.java index b6628dc5653..566cbbe5d72 100644 --- a/src/main/java/org/elasticsearch/cloud/gce/GceModule.java +++ b/src/main/java/org/elasticsearch/cloud/gce/GceModule.java @@ -20,14 +20,22 @@ package org.elasticsearch.cloud.gce; import org.elasticsearch.common.inject.AbstractModule; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; /** * */ public class GceModule extends AbstractModule { + private Class computeServiceClass; + + @Inject + public GceModule(Settings settings) { + this.computeServiceClass = settings.getAsClass("cloud.gce.api.impl", GceComputeServiceImpl.class); + } @Override protected void configure() { - bind(GceComputeService.class).to(GceComputeServiceImpl.class).asEagerSingleton(); + bind(GceComputeService.class).to(this.computeServiceClass).asEagerSingleton(); } } diff --git a/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java b/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java index 98fbb80960f..54342bdbbde 100644 --- a/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java +++ b/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java @@ -132,6 +132,7 @@ public class GceUnicastHostsProvider extends AbstractComponent implements Unicas String image = instance.getImage(); String status = instance.getStatus(); + logger.trace("gce instance {} with status {} found.", name, status); // We don't want to connect to TERMINATED status instances // See https://github.com/elasticsearch/elasticsearch-cloud-gce/issues/3 @@ -143,11 +144,14 @@ public class GceUnicastHostsProvider extends AbstractComponent implements Unicas // see if we need to filter by tag boolean filterByTag = false; if (tags.length > 0) { + logger.trace("start filtering instance {} with tags {}.", name, tags); if (instance.getTags() == null || instance.getTags().isEmpty()) { // If this instance have no tag, we filter it + logger.trace("no tags for this instance but we asked for tags. {} won't be part of the cluster.", name); filterByTag = true; } else { // check that all tags listed are there on the instance + logger.trace("comparing instance tags {} with tags filter {}.", instance.getTags().getItems(), tags); for (String tag : tags) { boolean found = false; for (String instancetag : instance.getTags().getItems()) { @@ -164,9 +168,11 @@ public class GceUnicastHostsProvider extends AbstractComponent implements Unicas } } if (filterByTag) { - logger.trace("filtering out instance {} based tags {}, not part of {}", name, tags, - instance.getTags().getItems()); + logger.trace("*** filtering out instance {} based tags {}, not part of {}", name, tags, + instance.getTags() == null ? "" : instance.getTags().getItems()); continue; + } else { + logger.trace("*** instance {} with tags {} is added to discovery", name, tags); } String ip_public = null; @@ -177,10 +183,12 @@ public class GceUnicastHostsProvider extends AbstractComponent implements Unicas for (NetworkInterface networkInterface : interfaces) { if (ip_public == null) { // Trying to get Public IP Address (For future use) - for (AccessConfig accessConfig : networkInterface.getAccessConfigs()) { - if (Strings.hasText(accessConfig.getNatIP())) { - ip_public = accessConfig.getNatIP(); - break; + if (networkInterface.getAccessConfigs() != null) { + for (AccessConfig accessConfig : networkInterface.getAccessConfigs()) { + if (Strings.hasText(accessConfig.getNatIP())) { + ip_public = accessConfig.getNatIP(); + break; + } } } } diff --git a/src/main/java/org/elasticsearch/plugin/cloud/gce/CloudGcePlugin.java b/src/main/java/org/elasticsearch/plugin/cloud/gce/CloudGcePlugin.java index 40be7338a09..f39e71c2cc2 100644 --- a/src/main/java/org/elasticsearch/plugin/cloud/gce/CloudGcePlugin.java +++ b/src/main/java/org/elasticsearch/plugin/cloud/gce/CloudGcePlugin.java @@ -19,7 +19,6 @@ package org.elasticsearch.plugin.cloud.gce; -import org.elasticsearch.cloud.gce.GceComputeServiceImpl; import org.elasticsearch.cloud.gce.GceModule; import org.elasticsearch.common.collect.Lists; import org.elasticsearch.common.component.LifecycleComponent; @@ -63,7 +62,7 @@ public class CloudGcePlugin extends AbstractPlugin { public Collection> services() { Collection> services = Lists.newArrayList(); if (settings.getAsBoolean("cloud.enabled", true)) { - services.add(GceComputeServiceImpl.class); +// services.add(GceComputeServiceImpl.class); } return services; } diff --git a/src/test/java/org/elasticsearch/cloud/gce/tests/GceAbstractTest.java b/src/test/java/org/elasticsearch/cloud/gce/tests/GceAbstractTest.java new file mode 100644 index 00000000000..e3716e264c8 --- /dev/null +++ b/src/test/java/org/elasticsearch/cloud/gce/tests/GceAbstractTest.java @@ -0,0 +1,105 @@ +/* + * Licensed to ElasticSearch under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.cloud.gce.tests; + +import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; +import org.elasticsearch.client.Client; +import org.elasticsearch.client.transport.TransportClient; +import org.elasticsearch.cloud.gce.GceComputeService; +import org.elasticsearch.common.io.FileSystemUtils; +import org.elasticsearch.common.settings.ImmutableSettings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.transport.InetSocketTransportAddress; +import org.elasticsearch.node.Node; +import org.elasticsearch.node.NodeBuilder; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; + +public abstract class GceAbstractTest { + + private static List nodes; + private Class mock; + + public GceAbstractTest(Class mock) { + // We want to inject the GCE API Mock + this.mock = mock; + } + + @Before + public void setUp() { + nodes = new ArrayList(); + + File dataDir = new File("./target/es/data"); + if(dataDir.exists()) { + FileSystemUtils.deleteRecursively(dataDir, true); + } + } + + @After + public void tearDown() { + // Cleaning nodes after test + for (Node node : nodes) { + node.close(); + } + } + + protected Client getClient(long nbOfNodes) { + // Create a TransportClient on node 1 and 2 + Settings settings = ImmutableSettings.settingsBuilder() + .put("cluster.name", "gce").build(); + + TransportClient client = new TransportClient(settings); + + for (int i = 9300; i < 9300+nbOfNodes; i++) { + client.addTransportAddress(new InetSocketTransportAddress("localhost", i)); + } + + return client; + } + + protected void checkNumberOfNodes(int expected) { + NodesInfoResponse nodeInfos = getClient(expected).admin().cluster().prepareNodesInfo().execute().actionGet(); + + Assert.assertNotNull(nodeInfos.getNodes()); + Assert.assertEquals(expected, nodeInfos.getNodes().length); + } + + protected void nodeBuilder(String filteredTags) { + ImmutableSettings.Builder builder = ImmutableSettings.settingsBuilder() + //.put("gateway.type", "local") + .put("path.data", "./target/es/data") + .put("path.logs", "./target/es/logs") + .put("path.work", "./target/es/work") + .put("cloud.gce.api.impl", mock) + .put("node.name", (nodes.size()+1) + "#" + mock.getSimpleName()); + if (filteredTags != null) { + builder.put("discovery.gce.tags", filteredTags); + } else { + builder.put("discovery.gce.tags", ""); + } + + Node node = NodeBuilder.nodeBuilder().settings(builder).node(); + nodes.add(node); + } +} diff --git a/src/test/java/org/elasticsearch/cloud/gce/tests/GceComputeServiceMock.java b/src/test/java/org/elasticsearch/cloud/gce/tests/GceComputeServiceAbstractMock.java similarity index 54% rename from src/test/java/org/elasticsearch/cloud/gce/tests/GceComputeServiceMock.java rename to src/test/java/org/elasticsearch/cloud/gce/tests/GceComputeServiceAbstractMock.java index 9d44d25e63b..e98b1498be6 100644 --- a/src/test/java/org/elasticsearch/cloud/gce/tests/GceComputeServiceMock.java +++ b/src/test/java/org/elasticsearch/cloud/gce/tests/GceComputeServiceAbstractMock.java @@ -20,6 +20,8 @@ package org.elasticsearch.cloud.gce.tests; import com.google.api.services.compute.model.Instance; +import com.google.api.services.compute.model.NetworkInterface; +import com.google.api.services.compute.model.Tags; import org.elasticsearch.ElasticSearchException; import org.elasticsearch.cloud.gce.GceComputeService; import org.elasticsearch.common.component.AbstractLifecycleComponent; @@ -27,22 +29,48 @@ import org.elasticsearch.common.settings.Settings; import java.util.ArrayList; import java.util.Collection; +import java.util.List; /** * */ -public class GceComputeServiceMock extends AbstractLifecycleComponent +public abstract class GceComputeServiceAbstractMock extends AbstractLifecycleComponent implements GceComputeService { - protected GceComputeServiceMock(Settings settings) { + protected abstract List> getTags(); + + protected GceComputeServiceAbstractMock(Settings settings) { super(settings); - logger.debug("Starting MOCK"); + logger.debug("starting GCE Api Mock with {} nodes:", getTags().size()); + for (List tags : getTags()) { + logger.debug(" - {}", tags); + } } @Override public Collection instances() { Collection instances = new ArrayList(); + // For each instance (item of tags) + for (List tags : getTags()) { + Instance instance = new Instance(); + instance.setName("Mock Node " + tags); + instance.setMachineType("Mock Type machine"); + instance.setImage("mock-image-type"); + instance.setStatus("STARTED"); + Tags instanceTags = new Tags(); + instanceTags.setItems(tags); + instance.setTags(instanceTags); + NetworkInterface networkInterface = new NetworkInterface(); + networkInterface.setNetworkIP("localhost"); + List networkInterfaces = new ArrayList(); + networkInterfaces.add(networkInterface); + instance.setNetworkInterfaces(networkInterfaces); + + instances.add(instance); + } + + return instances; } diff --git a/src/test/java/org/elasticsearch/cloud/gce/tests/GceComputeServiceTwoNodesDifferentTagsMock.java b/src/test/java/org/elasticsearch/cloud/gce/tests/GceComputeServiceTwoNodesDifferentTagsMock.java new file mode 100644 index 00000000000..7c59842ef98 --- /dev/null +++ b/src/test/java/org/elasticsearch/cloud/gce/tests/GceComputeServiceTwoNodesDifferentTagsMock.java @@ -0,0 +1,46 @@ +/* + * Licensed to ElasticSearch under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.cloud.gce.tests; + +import org.elasticsearch.common.collect.Lists; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; + +import java.util.ArrayList; +import java.util.List; + +/** + * + */ +public class GceComputeServiceTwoNodesDifferentTagsMock extends GceComputeServiceAbstractMock { + private static List> tags = Lists.newArrayList( + Lists.newArrayList("dev"), + Lists.newArrayList("elasticsearch","dev")); + + @Override + protected List> getTags() { + return tags; + } + + @Inject + protected GceComputeServiceTwoNodesDifferentTagsMock(Settings settings) { + super(settings); + } +} diff --git a/src/test/java/org/elasticsearch/cloud/gce/tests/GceComputeServiceTwoNodesSameTagsMock.java b/src/test/java/org/elasticsearch/cloud/gce/tests/GceComputeServiceTwoNodesSameTagsMock.java new file mode 100644 index 00000000000..90e1f32181d --- /dev/null +++ b/src/test/java/org/elasticsearch/cloud/gce/tests/GceComputeServiceTwoNodesSameTagsMock.java @@ -0,0 +1,48 @@ +/* + * Licensed to ElasticSearch under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.cloud.gce.tests; + +import org.elasticsearch.common.collect.Lists; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; + +import java.util.ArrayList; +import java.util.List; + +/** + * + */ +public class GceComputeServiceTwoNodesSameTagsMock extends GceComputeServiceAbstractMock { + private static List> tags = Lists.newArrayList( + Lists.newArrayList("elasticsearch","dev"), + Lists.newArrayList("elasticsearch","dev")); + + + @Override + protected List> getTags() { + return tags; + } + + @Inject + protected GceComputeServiceTwoNodesSameTagsMock(Settings settings) { + super(settings); + logger.debug("Starting Gce Mock {}", tags); + } +} diff --git a/src/test/java/org/elasticsearch/cloud/gce/tests/GceDifferentTagsTest.java b/src/test/java/org/elasticsearch/cloud/gce/tests/GceDifferentTagsTest.java new file mode 100644 index 00000000000..e8e7b438b18 --- /dev/null +++ b/src/test/java/org/elasticsearch/cloud/gce/tests/GceDifferentTagsTest.java @@ -0,0 +1,72 @@ +/* + * Licensed to ElasticSearch under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.cloud.gce.tests; + +import org.junit.Test; + +public class GceDifferentTagsTest extends GceAbstractTest { + + public GceDifferentTagsTest() { + super(GceComputeServiceTwoNodesDifferentTagsMock.class); + } + + @Test + public void two_nodes_should_run_no_tag() { + // Then we start our node for tests + nodeBuilder(null); + + // Let's start a second node + nodeBuilder(null); + + // We expect having 1 nodes as part of the cluster, let's test that + checkNumberOfNodes(2); + } + + @Test + public void one_node_should_run_two_tags() { + // Then we start our node for tests + nodeBuilder("elasticsearch,dev"); + + // Let's start a second node + nodeBuilder("elasticsearch,dev"); + + // We expect having 1 node as part of the cluster, let's test that + // TODO Fix that test as we don't filter yet on port, each node can see each other + checkNumberOfNodes(2); + + // Should be + // checkNumberOfNodes(1); + } + + @Test + public void one_node_should_be_filtered_one_tag() { + // Then we start our node for tests + nodeBuilder("elasticsearch"); + + // Let's start a second node + nodeBuilder("elasticsearch"); + + // We expect having 1 nodes as part of the cluster, let's test that + // TODO Fix that test as we don't filter yet on port, each node can see each other + checkNumberOfNodes(2); + + // Should be + // checkNumberOfNodes(1); + } +} diff --git a/src/test/java/org/elasticsearch/cloud/gce/tests/GceSameTagsTest.java b/src/test/java/org/elasticsearch/cloud/gce/tests/GceSameTagsTest.java new file mode 100644 index 00000000000..4da09b2dd2e --- /dev/null +++ b/src/test/java/org/elasticsearch/cloud/gce/tests/GceSameTagsTest.java @@ -0,0 +1,64 @@ +/* + * Licensed to ElasticSearch under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.cloud.gce.tests; + +import org.junit.Test; + +public class GceSameTagsTest extends GceAbstractTest { + + public GceSameTagsTest() { + super(GceComputeServiceTwoNodesSameTagsMock.class); + } + + @Test + public void two_nodes_should_run_no_tag() { + // Then we start our node for tests + nodeBuilder(null); + + // Let's start a second node + nodeBuilder(null); + + // We expect having 1 nodes as part of the cluster, let's test that + checkNumberOfNodes(2); + } + + @Test + public void two_nodes_should_run_two_tags() { + // Then we start our node for tests + nodeBuilder("elasticsearch,dev"); + + // Let's start a second node + nodeBuilder("elasticsearch,dev"); + + // We expect having 2 nodes as part of the cluster, let's test that + checkNumberOfNodes(2); + } + + @Test + public void two_nodes_should_run_one_tag() { + // Then we start our node for tests + nodeBuilder("elasticsearch"); + + // Let's start a second node + nodeBuilder("elasticsearch"); + + // We expect having 2 nodes as part of the cluster, let's test that + checkNumberOfNodes(2); + } +} diff --git a/src/test/java/org/elasticsearch/cloud/gce/tests/GceSimpleTest.java b/src/test/java/org/elasticsearch/cloud/gce/tests/GceSimpleTest.java deleted file mode 100644 index 90fcead8ccd..00000000000 --- a/src/test/java/org/elasticsearch/cloud/gce/tests/GceSimpleTest.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Licensed to ElasticSearch under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.cloud.gce.tests; - -import org.elasticsearch.common.io.FileSystemUtils; -import org.elasticsearch.common.settings.ImmutableSettings; -import org.elasticsearch.node.Node; -import org.elasticsearch.node.NodeBuilder; -import org.junit.Ignore; -import org.junit.Test; - -import java.io.File; - -/** - * This test can't work on a local machine as it needs to be launched from - * a GCE instance - * TODO: Mock GCE to simulate APIs REST Responses - */ -@Ignore -public class GceSimpleTest { - - @Test - public void launchNode() { - File dataDir = new File("./target/es/data"); - if(dataDir.exists()) { - FileSystemUtils.deleteRecursively(dataDir, true); - } - - // Then we start our node for tests - Node node = NodeBuilder - .nodeBuilder() - .settings( - ImmutableSettings.settingsBuilder() - .put("gateway.type", "local") - .put("path.data", "./target/es/data") - .put("path.logs", "./target/es/logs") - .put("path.work", "./target/es/work") - ).node(); - - // We wait now for the yellow (or green) status -// node.client().admin().cluster().prepareHealth().setWaitForYellowStatus().execute().actionGet(); - - } -} diff --git a/src/test/resources/elasticsearch.yml b/src/test/resources/elasticsearch.yml index e71e663bbe7..55606da022b 100644 --- a/src/test/resources/elasticsearch.yml +++ b/src/test/resources/elasticsearch.yml @@ -25,4 +25,6 @@ discovery: type: gce gce: - tags: elasticsearch, dev + tags: you_can_add_tags_here + +# Note that with tests, discovery.gce.tags will be replaced by test values From aaa4768131e8c3bed2e203901636617de7048dbc Mon Sep 17 00:00:00 2001 From: David Pilato Date: Mon, 8 Jul 2013 15:44:24 +0200 Subject: [PATCH 12/96] Add support for metadata `es_port`. If you run on a specific port, it will indicate it to GCE plugin. By default, `9300` will be used. --- README.md | 38 +++++++++++++++++++ .../gce/GceUnicastHostsProvider.java | 21 ++++++++-- .../tests/GceComputeServiceAbstractMock.java | 8 ++++ .../cloud/gce/tests/GceDifferentTagsTest.java | 12 +----- 4 files changed, 66 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 49fd6253797..2fce860f2f3 100644 --- a/README.md +++ b/README.md @@ -249,6 +249,44 @@ Then, define it in `elasticsearch.yml`: tags: elasticsearch, dev ``` +Changing default transport port +------------------------------- + +By default, elasticsearch GCE plugin assumes that you run elasticsearch on 9300 default port. +But you can specify the port value elasticsearch is meant to use using google compute engine metadata `es_port`: + +### When creating instance + +Add `--metadata=es_port:9301` option: + +```sh +# when creating first instance +gcutil --project=es-cloud addinstance myesnode1 --service_account_scope=compute-rw,storage-full --persistent_boot_disk --metadata=es_port:9301 + +# when creating an instance from an image +gcutil --project=es-cloud addinstance --image=elasticsearch-0-90-2 \ + --kernel=projects/google/global/kernels/gce-v20130603 myesnode2 \ + --zone europe-west1-a --machine_type f1-micro --service_account_scope=compute-rw \ + --persistent_boot_disk --metadata=es_port:9301 +``` + +### On a running instance + +```sh +# Get metadata fingerprint +gcutil --project=es-cloud getinstance myesnode1 --zone=europe-west1-a ++------------------------+---------------------------------------------------------------------------------------------------------+ +| property | value | ++------------------------+---------------------------------------------------------------------------------------------------------+ +| metadata | | +| fingerprint | 42WmSpB8rSM= | ++------------------------+---------------------------------------------------------------------------------------------------------+ + +# Use that fingerprint +gcutil --project=es-cloud setinstancemetadata myesnode1 --zone=europe-west1-a --metadata=es_port:9301 --fingerprint=42WmSpB8rSM= +``` + + Tips ---- diff --git a/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java b/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java index 54342bdbbde..d8ba7fe73f4 100644 --- a/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java +++ b/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java @@ -168,11 +168,11 @@ public class GceUnicastHostsProvider extends AbstractComponent implements Unicas } } if (filterByTag) { - logger.trace("*** filtering out instance {} based tags {}, not part of {}", name, tags, + logger.trace("filtering out instance {} based tags {}, not part of {}", name, tags, instance.getTags() == null ? "" : instance.getTags().getItems()); continue; } else { - logger.trace("*** instance {} with tags {} is added to discovery", name, tags); + logger.trace("instance {} with tags {} is added to discovery", name, tags); } String ip_public = null; @@ -207,7 +207,22 @@ public class GceUnicastHostsProvider extends AbstractComponent implements Unicas // We can ignore it in the list of DiscoveryNode logger.trace("current node found. Ignoring {} - {}", name, ip_private); } else { - TransportAddress[] addresses = transportService.addressesFromString(ip_private); + String address = ip_private; + // Test if we have es_port metadata defined here + if (instance.getMetadata() != null && instance.getMetadata().containsKey("es_port")) { + Object es_port = instance.getMetadata().get("es_port"); + logger.trace("es_port is defined with {}", es_port); + if (es_port instanceof String) { + address = address.concat(":").concat((String) es_port); + } else { + // Ignoring other values + logger.trace("es_port is instance of {}. Ignoring...", es_port.getClass().getName()); + } + } + + // ip_private is a single IP Address. We need to build a TransportAddress from it + TransportAddress[] addresses = transportService.addressesFromString(address); + // we only limit to 1 addresses, makes no sense to ping 100 ports for (int i = 0; (i < addresses.length && i < UnicastZenPing.LIMIT_PORTS_COUNT); i++) { logger.trace("adding {}, type {}, image {}, address {}, transport_address {}, status {}", name, type, diff --git a/src/test/java/org/elasticsearch/cloud/gce/tests/GceComputeServiceAbstractMock.java b/src/test/java/org/elasticsearch/cloud/gce/tests/GceComputeServiceAbstractMock.java index e98b1498be6..be37ac7588c 100644 --- a/src/test/java/org/elasticsearch/cloud/gce/tests/GceComputeServiceAbstractMock.java +++ b/src/test/java/org/elasticsearch/cloud/gce/tests/GceComputeServiceAbstractMock.java @@ -20,6 +20,7 @@ package org.elasticsearch.cloud.gce.tests; import com.google.api.services.compute.model.Instance; +import com.google.api.services.compute.model.Metadata; import com.google.api.services.compute.model.NetworkInterface; import com.google.api.services.compute.model.Tags; import org.elasticsearch.ElasticSearchException; @@ -52,6 +53,7 @@ public abstract class GceComputeServiceAbstractMock extends AbstractLifecycleCom Collection instances = new ArrayList(); // For each instance (item of tags) + int port = 9300; for (List tags : getTags()) { Instance instance = new Instance(); instance.setName("Mock Node " + tags); @@ -67,7 +69,13 @@ public abstract class GceComputeServiceAbstractMock extends AbstractLifecycleCom networkInterfaces.add(networkInterface); instance.setNetworkInterfaces(networkInterfaces); + // Add metadata es_port:930X where X is the instance number + Metadata metadata = new Metadata(); + metadata.put("es_port", "" + port); + instance.setMetadata(metadata); instances.add(instance); + + port++; } diff --git a/src/test/java/org/elasticsearch/cloud/gce/tests/GceDifferentTagsTest.java b/src/test/java/org/elasticsearch/cloud/gce/tests/GceDifferentTagsTest.java index e8e7b438b18..f555554876d 100644 --- a/src/test/java/org/elasticsearch/cloud/gce/tests/GceDifferentTagsTest.java +++ b/src/test/java/org/elasticsearch/cloud/gce/tests/GceDifferentTagsTest.java @@ -47,11 +47,7 @@ public class GceDifferentTagsTest extends GceAbstractTest { nodeBuilder("elasticsearch,dev"); // We expect having 1 node as part of the cluster, let's test that - // TODO Fix that test as we don't filter yet on port, each node can see each other - checkNumberOfNodes(2); - - // Should be - // checkNumberOfNodes(1); + checkNumberOfNodes(1); } @Test @@ -63,10 +59,6 @@ public class GceDifferentTagsTest extends GceAbstractTest { nodeBuilder("elasticsearch"); // We expect having 1 nodes as part of the cluster, let's test that - // TODO Fix that test as we don't filter yet on port, each node can see each other - checkNumberOfNodes(2); - - // Should be - // checkNumberOfNodes(1); + checkNumberOfNodes(1); } } From 6c0cc5f678a80aafe78c088fd8057c9ca5ac6d30 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Mon, 8 Jul 2013 22:44:18 +0200 Subject: [PATCH 13/96] Make documentation more readable --- README.md | 48 ++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 40 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 2fce860f2f3..3bb9731dede 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,9 @@ Before starting, you should have: ```sh -gcutil --project=es-cloud addinstance myesnode1 --service_account_scope=compute-rw,storage-full --persistent_boot_disk +gcutil --project=es-cloud addinstance myesnode1 \ + --service_account_scope=compute-rw,storage-full \ + --persistent_boot_disk ``` You will be asked to open a link in your browser. Login and allow access to listed services. @@ -128,7 +130,8 @@ Install the plugin: ```sh # Use Plugin Manager to install it -sudo /usr/share/elasticsearch/bin/plugin -install elasticsearch/elasticsearch-cloud-gce/1.0.0-SNAPSHOT +sudo /usr/share/elasticsearch/bin/plugin + --install elasticsearch/elasticsearch-cloud-gce/1.0.0 # Configure it: sudo vi /etc/elasticsearch/elasticsearch.yml @@ -152,6 +155,21 @@ Restart elasticsearch: sudo /etc/init.d/elasticsearch restart ``` +If anything goes wrong, you should check logs: + +```sh +tail -f /var/log/elasticsearch/elasticsearch.log +``` + +If needed, you can change log level to `TRACE` by modifying `sudo vi /etc/elasticsearch/logging.yml`: + +```yaml + # discovery + discovery.gce: TRACE +``` + + + ### Cloning your existing machine In order to build a cluster on many nodes, you can clone your configured instance to new nodes. @@ -190,6 +208,9 @@ gcutil listkernels --project es-cloud gcutil --project=es-cloud addimage elasticsearch-0-90-2 \ gs://esimage/e4686d7f5bf904a924ae0cfeb58d0827c6d5b966.image.tar.gz \ --preferred_kernel=projects/google/global/kernels/gce-v20130603 + +# If the previous command did not work for you, logout from your instance +# and launch the same command from your local machine. ``` ### Start new instances @@ -210,10 +231,12 @@ You can use [Google Cloud Console](https://cloud.google.com/console) or CLI to m ```sh # Stopping and removing instances -gcutil --project=es-cloud deleteinstance myesnode1 myesnode2 --zone=europe-west1-a +gcutil --project=es-cloud deleteinstance myesnode1 myesnode2 \ + --zone=europe-west1-a # Consider removing disk as well if you don't need them anymore -gcutil --project=es-cloud deletedisk boot-myesnode1 boot-myesnode2 --zone=europe-west1-a +gcutil --project=es-cloud deletedisk boot-myesnode1 boot-myesnode2 \ + --zone=europe-west1-a ``` Filtering by tags @@ -232,7 +255,9 @@ tag will resolve this issue. Add your tag when building the new instance: ```sh -gcutil --project=es-cloud addinstance myesnode1 --service_account_scope=compute-rw --persistent_boot_disk \ +gcutil --project=es-cloud addinstance myesnode1 \ + --service_account_scope=compute-rw \ + --persistent_boot_disk \ --tags=elasticsearch,dev ``` @@ -261,7 +286,10 @@ Add `--metadata=es_port:9301` option: ```sh # when creating first instance -gcutil --project=es-cloud addinstance myesnode1 --service_account_scope=compute-rw,storage-full --persistent_boot_disk --metadata=es_port:9301 +gcutil --project=es-cloud addinstance myesnode1 \ + --service_account_scope=compute-rw,storage-full \ + --persistent_boot_disk \ + --metadata=es_port:9301 # when creating an instance from an image gcutil --project=es-cloud addinstance --image=elasticsearch-0-90-2 \ @@ -274,7 +302,8 @@ gcutil --project=es-cloud addinstance --image=elasticsearch-0-90-2 \ ```sh # Get metadata fingerprint -gcutil --project=es-cloud getinstance myesnode1 --zone=europe-west1-a +gcutil --project=es-cloud getinstance myesnode1 \ + --zone=europe-west1-a +------------------------+---------------------------------------------------------------------------------------------------------+ | property | value | +------------------------+---------------------------------------------------------------------------------------------------------+ @@ -283,7 +312,10 @@ gcutil --project=es-cloud getinstance myesnode1 --zone=europe-west1-a +------------------------+---------------------------------------------------------------------------------------------------------+ # Use that fingerprint -gcutil --project=es-cloud setinstancemetadata myesnode1 --zone=europe-west1-a --metadata=es_port:9301 --fingerprint=42WmSpB8rSM= +gcutil --project=es-cloud setinstancemetadata myesnode1 \ + --zone=europe-west1-a \ + --metadata=es_port:9301 \ + --fingerprint=42WmSpB8rSM= ``` From 2814c51c24220d3376566edcc5e27ed99e820d54 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Tue, 9 Jul 2013 18:55:56 +0200 Subject: [PATCH 14/96] Changes after @imotov review. --- pom.xml | 17 ++++++++++++++++- .../org/elasticsearch/cloud/gce/GceModule.java | 8 +++++--- .../cloud/gce/tests/GceAbstractTest.java | 14 +++++++++----- src/test/resources/elasticsearch.yml | 9 +++++++++ 4 files changed, 39 insertions(+), 9 deletions(-) diff --git a/pom.xml b/pom.xml index bd83e6a476b..7804af30f6c 100644 --- a/pom.xml +++ b/pom.xml @@ -113,7 +113,22 @@ governing permissions and limitations under the License. --> true - + + + src/test/resources + true + + elasticsearch.yml + + + + src/test/resources + false + + log4j.xml + + + org.apache.maven.plugins diff --git a/src/main/java/org/elasticsearch/cloud/gce/GceModule.java b/src/main/java/org/elasticsearch/cloud/gce/GceModule.java index 566cbbe5d72..d5c513adeae 100644 --- a/src/main/java/org/elasticsearch/cloud/gce/GceModule.java +++ b/src/main/java/org/elasticsearch/cloud/gce/GceModule.java @@ -27,15 +27,17 @@ import org.elasticsearch.common.settings.Settings; * */ public class GceModule extends AbstractModule { - private Class computeServiceClass; + private Settings settings; @Inject public GceModule(Settings settings) { - this.computeServiceClass = settings.getAsClass("cloud.gce.api.impl", GceComputeServiceImpl.class); + this.settings = settings; } @Override protected void configure() { - bind(GceComputeService.class).to(this.computeServiceClass).asEagerSingleton(); + bind(GceComputeService.class) + .to(settings.getAsClass("cloud.gce.api.impl", GceComputeServiceImpl.class)) + .asEagerSingleton(); } } diff --git a/src/test/java/org/elasticsearch/cloud/gce/tests/GceAbstractTest.java b/src/test/java/org/elasticsearch/cloud/gce/tests/GceAbstractTest.java index e3716e264c8..7e0333b8ed0 100644 --- a/src/test/java/org/elasticsearch/cloud/gce/tests/GceAbstractTest.java +++ b/src/test/java/org/elasticsearch/cloud/gce/tests/GceAbstractTest.java @@ -25,9 +25,11 @@ import org.elasticsearch.cloud.gce.GceComputeService; import org.elasticsearch.common.io.FileSystemUtils; import org.elasticsearch.common.settings.ImmutableSettings; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.transport.InetSocketTransportAddress; +import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.node.Node; import org.elasticsearch.node.NodeBuilder; +import org.elasticsearch.node.internal.InternalNode; +import org.elasticsearch.transport.netty.NettyTransport; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -64,22 +66,24 @@ public abstract class GceAbstractTest { } } - protected Client getClient(long nbOfNodes) { + protected Client getClient() { // Create a TransportClient on node 1 and 2 Settings settings = ImmutableSettings.settingsBuilder() .put("cluster.name", "gce").build(); TransportClient client = new TransportClient(settings); - for (int i = 9300; i < 9300+nbOfNodes; i++) { - client.addTransportAddress(new InetSocketTransportAddress("localhost", i)); + for (Node node : nodes) { + NettyTransport nettyTransport = ((InternalNode) node).injector().getInstance(NettyTransport.class); + TransportAddress transportAddress = nettyTransport.boundAddress().publishAddress(); + client.addTransportAddress(transportAddress); } return client; } protected void checkNumberOfNodes(int expected) { - NodesInfoResponse nodeInfos = getClient(expected).admin().cluster().prepareNodesInfo().execute().actionGet(); + NodesInfoResponse nodeInfos = getClient().admin().cluster().prepareNodesInfo().execute().actionGet(); Assert.assertNotNull(nodeInfos.getNodes()); Assert.assertEquals(expected, nodeInfos.getNodes().length); diff --git a/src/test/resources/elasticsearch.yml b/src/test/resources/elasticsearch.yml index 55606da022b..e6db977d4e3 100644 --- a/src/test/resources/elasticsearch.yml +++ b/src/test/resources/elasticsearch.yml @@ -28,3 +28,12 @@ tags: you_can_add_tags_here # Note that with tests, discovery.gce.tags will be replaced by test values + +# We set path here to use maven target dir + path: + data: ${project.build.testOutputDirectory}/es/data + logs: ${project.build.testOutputDirectory}/es/logs + work: ${project.build.testOutputDirectory}/es/work + +# We don't need http layer for tests + http.enabled: false From c57903dbaf2b576e0128acce52b56abf7e26871a Mon Sep 17 00:00:00 2001 From: David Pilato Date: Tue, 23 Jul 2013 13:17:19 +0200 Subject: [PATCH 15/96] Cleaning code before release --- .../cloud/gce/GceComputeServiceImpl.java | 2 +- .../discovery/gce/GceUnicastHostsProvider.java | 12 +++++------- .../cloud/gce/tests/GceAbstractTest.java | 1 + src/test/resources/log4j.xml | 10 +++++----- 4 files changed, 12 insertions(+), 13 deletions(-) diff --git a/src/main/java/org/elasticsearch/cloud/gce/GceComputeServiceImpl.java b/src/main/java/org/elasticsearch/cloud/gce/GceComputeServiceImpl.java index 38290bafe95..3b6c0a38ebc 100644 --- a/src/main/java/org/elasticsearch/cloud/gce/GceComputeServiceImpl.java +++ b/src/main/java/org/elasticsearch/cloud/gce/GceComputeServiceImpl.java @@ -94,7 +94,7 @@ public class GceComputeServiceImpl extends AbstractLifecycleComponent - + - - + + - - + + From e0a99df201e74eb79c12d29a41cc3c85d2bd350d Mon Sep 17 00:00:00 2001 From: David Pilato Date: Tue, 23 Jul 2013 12:48:13 +0200 Subject: [PATCH 16/96] prepare release elasticsearch-cloud-gce-1.0.0 --- README.md | 12 +++++++----- pom.xml | 21 +-------------------- src/test/resources/log4j.xml | 2 +- 3 files changed, 9 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 3bb9731dede..7d808e55fd7 100644 --- a/README.md +++ b/README.md @@ -5,11 +5,13 @@ The GCE Cloud plugin allows to use GCE API for the unicast discovery mechanism. In order to install the plugin, simply run: `bin/plugin -install elasticsearch/elasticsearch-cloud-gce/1.0.0-SNAPSHOT`. - ---------------------------------------------- - | GCE Cloud Plugin | Elasticsearch | - ---------------------------------------------- - | 1.0.0-SNAPSHOT (master) | 0.90 -> master | - ---------------------------------------------- + ------------------------------------------------------------- + | GCE Cloud Plugin | Elasticsearch | Release date | + ------------------------------------------------------------- + | 1.1.0-SNAPSHOT (master) | 0.90 -> master | | + ------------------------------------------------------------- + | 1.0.0 | 0.90 -> master | 2013-07-23 | + ------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 7804af30f6c..7c876a34652 100644 --- a/pom.xml +++ b/pom.xml @@ -17,7 +17,7 @@ governing permissions and limitations under the License. --> 4.0.0 org.elasticsearch elasticsearch-cloud-gce - 1.0.0-SNAPSHOT + 1.0.0 Google Compute Engine Cloud plugin for ElasticSearch 2013 @@ -176,25 +176,6 @@ governing permissions and limitations under the License. --> - - org.codehaus.mojo - wagon-maven-plugin - 1.0-beta-4 - - elasticsearch.download - ${project.build.directory}/releases/${project.build.finalName}.zip - s3://test.elasticsearch.org/elasticsearch/${project.artifactId}/ - - - - upload-zip-to-server - deploy - - upload-single - - - - diff --git a/src/test/resources/log4j.xml b/src/test/resources/log4j.xml index ae7b83715c7..0c56ac9498c 100644 --- a/src/test/resources/log4j.xml +++ b/src/test/resources/log4j.xml @@ -32,7 +32,7 @@ governing permissions and limitations under the License. --> - + From d8f308c2edc10754e4638b009f98462dacd15ccf Mon Sep 17 00:00:00 2001 From: David Pilato Date: Tue, 23 Jul 2013 13:40:09 +0200 Subject: [PATCH 17/96] prepare for next development iteration --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 7c876a34652..39dc7d4ff53 100644 --- a/pom.xml +++ b/pom.xml @@ -17,7 +17,7 @@ governing permissions and limitations under the License. --> 4.0.0 org.elasticsearch elasticsearch-cloud-gce - 1.0.0 + 1.1.0-SNAPSHOT Google Compute Engine Cloud plugin for ElasticSearch 2013 From 27521f61f393d6f1baf86387cfeebd84490eeb3a Mon Sep 17 00:00:00 2001 From: David Pilato Date: Tue, 23 Jul 2013 13:41:48 +0200 Subject: [PATCH 18/96] Fix version number in `plugin -install` command --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7d808e55fd7..5b4dc0c87b2 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ Google Compute Engine Cloud Plugin for Elasticsearch The GCE Cloud plugin allows to use GCE API for the unicast discovery mechanism. -In order to install the plugin, simply run: `bin/plugin -install elasticsearch/elasticsearch-cloud-gce/1.0.0-SNAPSHOT`. +In order to install the plugin, simply run: `bin/plugin -install elasticsearch/elasticsearch-cloud-gce/1.0.0`. ------------------------------------------------------------- | GCE Cloud Plugin | Elasticsearch | Release date | From 615b1f63e5dd9cac256847316bf9941ee05e6a9b Mon Sep 17 00:00:00 2001 From: David Pilato Date: Tue, 23 Jul 2013 17:55:55 +0200 Subject: [PATCH 19/96] Moving log level to debug We don't want to have too much INFO log level. So we move some parts to log.debug. Closes #4. --- .../elasticsearch/discovery/gce/GceUnicastHostsProvider.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java b/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java index bb73c8ff015..b163a88fe85 100644 --- a/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java +++ b/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java @@ -105,7 +105,7 @@ public class GceUnicastHostsProvider extends AbstractComponent implements Unicas } lastRefresh = System.currentTimeMillis(); } - logger.info("start building nodes list using GCE API"); + logger.debug("start building nodes list using GCE API"); cachedDiscoNodes = Lists.newArrayList(); String ipAddress = null; @@ -236,7 +236,7 @@ public class GceUnicastHostsProvider extends AbstractComponent implements Unicas logger.trace("Exception caught during discovery", e); } - logger.info("{} node(s) added", cachedDiscoNodes.size()); + logger.debug("{} node(s) added", cachedDiscoNodes.size()); logger.debug("using dynamic discovery nodes {}", cachedDiscoNodes); return cachedDiscoNodes; From b04b24c91a46f99faa302a18d3911be7a2e9bf91 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Fri, 9 Aug 2013 09:39:14 +0200 Subject: [PATCH 20/96] Fix path for tests --- src/test/resources/elasticsearch.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/test/resources/elasticsearch.yml b/src/test/resources/elasticsearch.yml index e6db977d4e3..1d13d3e65b9 100644 --- a/src/test/resources/elasticsearch.yml +++ b/src/test/resources/elasticsearch.yml @@ -31,9 +31,9 @@ # We set path here to use maven target dir path: - data: ${project.build.testOutputDirectory}/es/data - logs: ${project.build.testOutputDirectory}/es/logs - work: ${project.build.testOutputDirectory}/es/work + data: ${project.build.directory}/es/data + logs: ${project.build.directory}/es/logs + work: ${project.build.directory}/es/work # We don't need http layer for tests http.enabled: false From ed562689f7098417bbee52260a7f846ca8d0d08c Mon Sep 17 00:00:00 2001 From: David Pilato Date: Fri, 9 Aug 2013 09:58:38 +0200 Subject: [PATCH 21/96] Update to Elasticsearch 0.90.3. Closes #5. --- pom.xml | 2 +- .../java/org/elasticsearch/discovery/gce/GceDiscovery.java | 3 ++- .../elasticsearch/discovery/gce/GceUnicastHostsProvider.java | 3 ++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 39dc7d4ff53..dc7e33c0df4 100644 --- a/pom.xml +++ b/pom.xml @@ -47,7 +47,7 @@ governing permissions and limitations under the License. --> - 0.90.2 + 0.90.3 v1beta15-rev3-1.15.0-rc 1.15.0-rc diff --git a/src/main/java/org/elasticsearch/discovery/gce/GceDiscovery.java b/src/main/java/org/elasticsearch/discovery/gce/GceDiscovery.java index 717398e97a6..199b9799c8e 100755 --- a/src/main/java/org/elasticsearch/discovery/gce/GceDiscovery.java +++ b/src/main/java/org/elasticsearch/discovery/gce/GceDiscovery.java @@ -19,6 +19,7 @@ package org.elasticsearch.discovery.gce; +import org.elasticsearch.Version; import org.elasticsearch.cloud.gce.GceComputeService; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterService; @@ -45,7 +46,7 @@ public class GceDiscovery extends ZenDiscovery { ClusterService clusterService, NodeSettingsService nodeSettingsService, ZenPingService pingService, DiscoveryNodeService discoveryNodeService, GceComputeService gceComputeService, NetworkService networkService) { - super(settings, clusterName, threadPool, transportService, clusterService, nodeSettingsService, discoveryNodeService, pingService); + super(settings, clusterName, threadPool, transportService, clusterService, nodeSettingsService, discoveryNodeService, pingService, Version.CURRENT); if (settings.getAsBoolean("cloud.enabled", true)) { ImmutableList zenPings = pingService.zenPings(); UnicastZenPing unicastZenPing = null; diff --git a/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java b/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java index b163a88fe85..cf07317f5db 100644 --- a/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java +++ b/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java @@ -22,6 +22,7 @@ package org.elasticsearch.discovery.gce; import com.google.api.services.compute.model.AccessConfig; import com.google.api.services.compute.model.Instance; import com.google.api.services.compute.model.NetworkInterface; +import org.elasticsearch.Version; import org.elasticsearch.cloud.gce.GceComputeService; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.Strings; @@ -223,7 +224,7 @@ public class GceUnicastHostsProvider extends AbstractComponent implements Unicas for (int i = 0; (i < addresses.length && i < UnicastZenPing.LIMIT_PORTS_COUNT); i++) { logger.trace("adding {}, type {}, image {}, address {}, transport_address {}, status {}", name, type, image, ip_private, addresses[i], status); - cachedDiscoNodes.add(new DiscoveryNode("#cloud-" + name + "-" + i, addresses[i])); + cachedDiscoNodes.add(new DiscoveryNode("#cloud-" + name + "-" + i, addresses[i], Version.CURRENT)); } } } catch (Exception e) { From 1930c09eb2b0022bdd5543d77642fba8df552015 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Fri, 9 Aug 2013 10:00:44 +0200 Subject: [PATCH 22/96] prepare release elasticsearch-cloud-gce-1.1.0 --- README.md | 10 ++++++---- pom.xml | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 5b4dc0c87b2..814cf1cf369 100644 --- a/README.md +++ b/README.md @@ -3,14 +3,16 @@ Google Compute Engine Cloud Plugin for Elasticsearch The GCE Cloud plugin allows to use GCE API for the unicast discovery mechanism. -In order to install the plugin, simply run: `bin/plugin -install elasticsearch/elasticsearch-cloud-gce/1.0.0`. +In order to install the plugin, simply run: `bin/plugin -install elasticsearch/elasticsearch-cloud-gce/1.1.0`. ------------------------------------------------------------- | GCE Cloud Plugin | Elasticsearch | Release date | ------------------------------------------------------------- - | 1.1.0-SNAPSHOT (master) | 0.90 -> master | | + | 1.2.0-SNAPSHOT (master) | 0.90 -> master | | ------------------------------------------------------------- - | 1.0.0 | 0.90 -> master | 2013-07-23 | + | 1.1.0 | 0.90.3 -> master | 2013-08-09 | + ------------------------------------------------------------- + | 1.0.0 | 0.90 -> 0.90.2 | 2013-07-23 | ------------------------------------------------------------- @@ -133,7 +135,7 @@ Install the plugin: ```sh # Use Plugin Manager to install it sudo /usr/share/elasticsearch/bin/plugin - --install elasticsearch/elasticsearch-cloud-gce/1.0.0 + --install elasticsearch/elasticsearch-cloud-gce/1.1.0 # Configure it: sudo vi /etc/elasticsearch/elasticsearch.yml diff --git a/pom.xml b/pom.xml index dc7e33c0df4..cdaf5d6c12a 100644 --- a/pom.xml +++ b/pom.xml @@ -17,7 +17,7 @@ governing permissions and limitations under the License. --> 4.0.0 org.elasticsearch elasticsearch-cloud-gce - 1.1.0-SNAPSHOT + 1.1.0 Google Compute Engine Cloud plugin for ElasticSearch 2013 From e2f6e5c15214c018aed80cb684bc558798238ce3 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Fri, 9 Aug 2013 10:08:33 +0200 Subject: [PATCH 23/96] prepare for next development iteration --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index cdaf5d6c12a..d446ae1787a 100644 --- a/pom.xml +++ b/pom.xml @@ -17,7 +17,7 @@ governing permissions and limitations under the License. --> 4.0.0 org.elasticsearch elasticsearch-cloud-gce - 1.1.0 + 1.2.0-SNAPSHOT Google Compute Engine Cloud plugin for ElasticSearch 2013 From 866880e26d1265bcc29a5b38a8a8b207df32f0ae Mon Sep 17 00:00:00 2001 From: David Pilato Date: Sat, 10 Aug 2013 18:43:13 +0200 Subject: [PATCH 24/96] Move version table to markdown format --- README.md | 37 ++++++++++++++++++++++++++----------- 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 814cf1cf369..b2138a6e56b 100644 --- a/README.md +++ b/README.md @@ -5,17 +5,32 @@ The GCE Cloud plugin allows to use GCE API for the unicast discovery mechanism. In order to install the plugin, simply run: `bin/plugin -install elasticsearch/elasticsearch-cloud-gce/1.1.0`. - ------------------------------------------------------------- - | GCE Cloud Plugin | Elasticsearch | Release date | - ------------------------------------------------------------- - | 1.2.0-SNAPSHOT (master) | 0.90 -> master | | - ------------------------------------------------------------- - | 1.1.0 | 0.90.3 -> master | 2013-08-09 | - ------------------------------------------------------------- - | 1.0.0 | 0.90 -> 0.90.2 | 2013-07-23 | - ------------------------------------------------------------- - - + + + + + + + + + + + + + + + + + + + + + + + + + +
GCE Cloud PluginElasticSearchRelease date
1.2.0-SNAPSHOT (master)0.90.3 -> master
1.1.00.90.3 -> master2013-08-09
1.0.00.90 -> 0.90.22013-07-23
Google Compute Engine Virtual Machine Discovery =============================== From 0c0d4b9731df1070ddbaf2ec4f4721f418bd4ab5 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Wed, 21 Aug 2013 11:55:59 +0200 Subject: [PATCH 25/96] Create CONTRIBUTING.md --- CONTRIBUTING.md | 98 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 98 insertions(+) create mode 100644 CONTRIBUTING.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000000..54c5c543c89 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,98 @@ +Contributing to elasticsearch +============================= + +Elasticsearch is an open source project and we love to receive contributions from our community — you! There are many ways to contribute, from writing tutorials or blog posts, improving the documentation, submitting bug reports and feature requests or writing code which can be incorporated into Elasticsearch itself. + +Bug reports +----------- + +If you think you have found a bug in Elasticsearch, first make sure that you are testing against the [latest version of Elasticsearch](http://www.elasticsearch.org/download/) - your issue may already have been fixed. If not, search our [issues list](https://github.com/elasticsearch/elasticsearch/issues) on GitHub in case a similar issue has already been opened. + +It is very helpful if you can prepare a reproduction of the bug. In other words, provide a small test case which we can run to confirm your bug. It makes it easier to find the problem and to fix it. Test cases should be provided as `curl` commands which we can copy and paste into a terminal to run it locally, for example: + +```sh +# delete the index +curl -XDELETE localhost:9200/test + +# insert a document +curl -XPUT localhost:9200/test/test/1 -d '{ + "title": "test document" +}' + +# this should return XXXX but instead returns YYY +curl .... +``` + +Provide as much information as you can. You may think that the problem lies with your query, when actually it depends on how your data is indexed. The easier it is for us to recreate your problem, the faster it is likely to be fixed. + +Feature requests +---------------- + +If you find yourself wishing for a feature that doesn't exist in Elasticsearch, you are probably not alone. There are bound to be others out there with similar needs. Many of the features that Elasticsearch has today have been added because our users saw the need. +Open an issue on our [issues list](https://github.com/elasticsearch/elasticsearch/issues) on GitHub which describes the feature you would like to see, why you need it, and how it should work. + +Contributing code and documentation changes +------------------------------------------- + +If you have a bugfix or new feature that you would like to contribute to Elasticsearch, please find or open an issue about it first. Talk about what you would like to do. It may be that somebody is already working on it, or that there are particular issues that you should know about before implementing the change. + +We enjoy working with contributors to get their code accepted. There are many approaches to fixing a problem and it is important to find the best approach before writing too much code. + +The process for contributing to any of the [Elasticsearch repositories](https://github.com/elasticsearch/) is similar. Details for individual projects can be found below. + +### Fork and clone the repository + +You will need to fork the main Elasticsearch code or documentation repository and clone it to your local machine. See +[github help page](https://help.github.com/articles/fork-a-repo) for help. + +Further instructions for specific projects are given below. + +### Submitting your changes + +Once your changes and tests are ready to submit for review: + +1. Test your changes +Run the test suite to make sure that nothing is broken. + +2. Sign the Contributor License Agreement +Please make sure you have signed our [Contributor License Agreement](http://www.elasticsearch.org/contributor-agreement/). We are not asking you to assign copyright to us, but to give us the right to distribute your code without restriction. We ask this of all contributors in order to assure our users of the origin and continuing existence of the code. You only need to sign the CLA once. + +3. Rebase your changes +Update your local repository with the most recent code from the main Elasticsearch repository, and rebase your branch on top of the latest master branch. We prefer your changes to be squashed into a single commit. + +4. Submit a pull request +Push your local changes to your forked copy of the repository and [submit a pull request](https://help.github.com/articles/using-pull-requests). In the pull request, describe what your changes do and mention the number of the issue where discussion has taken place, eg "Closes #123". + +Then sit back and wait. There will probably be discussion about the pull request and, if any changes are needed, we would love to work with you to get your pull request merged into Elasticsearch. + + +Contributing to the Elasticsearch plugin +---------------------------------------- + +**Repository:** [https://github.com/elasticsearch/elasticsearch-cloud-gce](https://github.com/elasticsearch/elasticsearch-cloud-gce) + +Make sure you have [Maven](http://maven.apache.org) installed, as Elasticsearch uses it as its build system. Integration with IntelliJ and Eclipse should work out of the box. Eclipse users can automatically configure their IDE by running `mvn eclipse:eclipse` and then importing the project into their workspace: `File > Import > Existing project into workspace`. + +Please follow these formatting guidelines: + +* Java indent is 4 spaces +* Line width is 140 characters +* The rest is left to Java coding standards +* Disable “auto-format on save” to prevent unnecessary format changes. This makes reviews much harder as it generates unnecessary formatting changes. If your IDE supports formatting only modified chunks that is fine to do. + +To create a distribution from the source, simply run: + +```sh +cd elasticsearch-cloud-gce/ +mvn clean package -DskipTests +``` + +You will find the newly built packages under: `./target/releases/`. + +Before submitting your changes, run the test suite to make sure that nothing is broken, with: + +```sh +mvn clean test +``` + +Source: [Contributing to elasticsearch](http://www.elasticsearch.org/contributing-to-elasticsearch/) From cc786941f4f18290dcceb3cff3191fd3548ce69a Mon Sep 17 00:00:00 2001 From: David Pilato Date: Mon, 13 Jan 2014 14:56:26 +0100 Subject: [PATCH 26/96] update headers --- README.md | 33 ++++--------------- .../cloud/gce/GceComputeService.java | 14 ++++---- .../cloud/gce/GceComputeServiceImpl.java | 14 ++++---- .../elasticsearch/cloud/gce/GceModule.java | 14 ++++---- .../cloud/gce/GceSettingsFilter.java | 14 ++++---- .../discovery/gce/GceDiscovery.java | 14 ++++---- .../discovery/gce/GceDiscoveryModule.java | 14 ++++---- .../gce/GceUnicastHostsProvider.java | 14 ++++---- .../plugin/cloud/gce/CloudGcePlugin.java | 14 ++++---- .../cloud/gce/tests/GceAbstractTest.java | 15 +++++---- .../tests/GceComputeServiceAbstractMock.java | 14 ++++---- ...mputeServiceTwoNodesDifferentTagsMock.java | 14 ++++---- ...GceComputeServiceTwoNodesSameTagsMock.java | 14 ++++---- .../cloud/gce/tests/GceDifferentTagsTest.java | 15 +++++---- .../cloud/gce/tests/GceSameTagsTest.java | 15 +++++---- 15 files changed, 108 insertions(+), 124 deletions(-) diff --git a/README.md b/README.md index b2138a6e56b..d59630ca240 100644 --- a/README.md +++ b/README.md @@ -5,32 +5,13 @@ The GCE Cloud plugin allows to use GCE API for the unicast discovery mechanism. In order to install the plugin, simply run: `bin/plugin -install elasticsearch/elasticsearch-cloud-gce/1.1.0`. - - - - - - - - - - - - - - - - - - - - - - - - - -
GCE Cloud PluginElasticSearchRelease date
1.2.0-SNAPSHOT (master)0.90.3 -> master
1.1.00.90.3 -> master2013-08-09
1.0.00.90 -> 0.90.22013-07-23
+| GCE Cloud Plugin | elasticsearch | Release date | +|----------------------------|---------------------|:------------:| +| 2.0.0.RC1-SNAPSHOT (master)| 1.0.0.RC1 -> master | | +| 1.2.0-SNAPSHOT (1.x) | 0.90.3 -> 0.90 | | +| 1.1.0 | 0.90.3 -> 0.90 | 2013-08-09 | +| 1.0.0 | 0.90.0 -> 0.90.2 | 2013-07-23 | + Google Compute Engine Virtual Machine Discovery =============================== diff --git a/src/main/java/org/elasticsearch/cloud/gce/GceComputeService.java b/src/main/java/org/elasticsearch/cloud/gce/GceComputeService.java index 262aa8834c2..cb0e35654a7 100644 --- a/src/main/java/org/elasticsearch/cloud/gce/GceComputeService.java +++ b/src/main/java/org/elasticsearch/cloud/gce/GceComputeService.java @@ -1,11 +1,11 @@ /* - * Licensed to ElasticSearch under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * diff --git a/src/main/java/org/elasticsearch/cloud/gce/GceComputeServiceImpl.java b/src/main/java/org/elasticsearch/cloud/gce/GceComputeServiceImpl.java index 3b6c0a38ebc..8ff193909d9 100644 --- a/src/main/java/org/elasticsearch/cloud/gce/GceComputeServiceImpl.java +++ b/src/main/java/org/elasticsearch/cloud/gce/GceComputeServiceImpl.java @@ -1,11 +1,11 @@ /* - * Licensed to ElasticSearch under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * diff --git a/src/main/java/org/elasticsearch/cloud/gce/GceModule.java b/src/main/java/org/elasticsearch/cloud/gce/GceModule.java index d5c513adeae..870698c3778 100644 --- a/src/main/java/org/elasticsearch/cloud/gce/GceModule.java +++ b/src/main/java/org/elasticsearch/cloud/gce/GceModule.java @@ -1,11 +1,11 @@ /* - * Licensed to ElasticSearch under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * diff --git a/src/main/java/org/elasticsearch/cloud/gce/GceSettingsFilter.java b/src/main/java/org/elasticsearch/cloud/gce/GceSettingsFilter.java index 64aa69a943c..7f90429ced9 100644 --- a/src/main/java/org/elasticsearch/cloud/gce/GceSettingsFilter.java +++ b/src/main/java/org/elasticsearch/cloud/gce/GceSettingsFilter.java @@ -1,11 +1,11 @@ /* - * Licensed to ElasticSearch under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * diff --git a/src/main/java/org/elasticsearch/discovery/gce/GceDiscovery.java b/src/main/java/org/elasticsearch/discovery/gce/GceDiscovery.java index 199b9799c8e..cf82ac1f78f 100755 --- a/src/main/java/org/elasticsearch/discovery/gce/GceDiscovery.java +++ b/src/main/java/org/elasticsearch/discovery/gce/GceDiscovery.java @@ -1,11 +1,11 @@ /* - * Licensed to ElasticSearch under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * diff --git a/src/main/java/org/elasticsearch/discovery/gce/GceDiscoveryModule.java b/src/main/java/org/elasticsearch/discovery/gce/GceDiscoveryModule.java index b7970d9ba97..2dcb65789e9 100644 --- a/src/main/java/org/elasticsearch/discovery/gce/GceDiscoveryModule.java +++ b/src/main/java/org/elasticsearch/discovery/gce/GceDiscoveryModule.java @@ -1,11 +1,11 @@ /* - * Licensed to ElasticSearch under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * diff --git a/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java b/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java index cf07317f5db..15c498a752b 100644 --- a/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java +++ b/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java @@ -1,11 +1,11 @@ /* - * Licensed to ElasticSearch under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * diff --git a/src/main/java/org/elasticsearch/plugin/cloud/gce/CloudGcePlugin.java b/src/main/java/org/elasticsearch/plugin/cloud/gce/CloudGcePlugin.java index f39e71c2cc2..b6f3cf0216f 100644 --- a/src/main/java/org/elasticsearch/plugin/cloud/gce/CloudGcePlugin.java +++ b/src/main/java/org/elasticsearch/plugin/cloud/gce/CloudGcePlugin.java @@ -1,11 +1,11 @@ /* - * Licensed to ElasticSearch under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * diff --git a/src/test/java/org/elasticsearch/cloud/gce/tests/GceAbstractTest.java b/src/test/java/org/elasticsearch/cloud/gce/tests/GceAbstractTest.java index bcf2eee5ba1..05804ffdd73 100644 --- a/src/test/java/org/elasticsearch/cloud/gce/tests/GceAbstractTest.java +++ b/src/test/java/org/elasticsearch/cloud/gce/tests/GceAbstractTest.java @@ -1,11 +1,11 @@ /* - * Licensed to ElasticSearch under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -16,6 +16,7 @@ * specific language governing permissions and limitations * under the License. */ + package org.elasticsearch.cloud.gce.tests; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; diff --git a/src/test/java/org/elasticsearch/cloud/gce/tests/GceComputeServiceAbstractMock.java b/src/test/java/org/elasticsearch/cloud/gce/tests/GceComputeServiceAbstractMock.java index be37ac7588c..09b321bb725 100644 --- a/src/test/java/org/elasticsearch/cloud/gce/tests/GceComputeServiceAbstractMock.java +++ b/src/test/java/org/elasticsearch/cloud/gce/tests/GceComputeServiceAbstractMock.java @@ -1,11 +1,11 @@ /* - * Licensed to ElasticSearch under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * diff --git a/src/test/java/org/elasticsearch/cloud/gce/tests/GceComputeServiceTwoNodesDifferentTagsMock.java b/src/test/java/org/elasticsearch/cloud/gce/tests/GceComputeServiceTwoNodesDifferentTagsMock.java index 7c59842ef98..5981342909c 100644 --- a/src/test/java/org/elasticsearch/cloud/gce/tests/GceComputeServiceTwoNodesDifferentTagsMock.java +++ b/src/test/java/org/elasticsearch/cloud/gce/tests/GceComputeServiceTwoNodesDifferentTagsMock.java @@ -1,11 +1,11 @@ /* - * Licensed to ElasticSearch under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * diff --git a/src/test/java/org/elasticsearch/cloud/gce/tests/GceComputeServiceTwoNodesSameTagsMock.java b/src/test/java/org/elasticsearch/cloud/gce/tests/GceComputeServiceTwoNodesSameTagsMock.java index 90e1f32181d..0dea3b9eb2f 100644 --- a/src/test/java/org/elasticsearch/cloud/gce/tests/GceComputeServiceTwoNodesSameTagsMock.java +++ b/src/test/java/org/elasticsearch/cloud/gce/tests/GceComputeServiceTwoNodesSameTagsMock.java @@ -1,11 +1,11 @@ /* - * Licensed to ElasticSearch under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * diff --git a/src/test/java/org/elasticsearch/cloud/gce/tests/GceDifferentTagsTest.java b/src/test/java/org/elasticsearch/cloud/gce/tests/GceDifferentTagsTest.java index f555554876d..155f99ecaf4 100644 --- a/src/test/java/org/elasticsearch/cloud/gce/tests/GceDifferentTagsTest.java +++ b/src/test/java/org/elasticsearch/cloud/gce/tests/GceDifferentTagsTest.java @@ -1,11 +1,11 @@ /* - * Licensed to ElasticSearch under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -16,6 +16,7 @@ * specific language governing permissions and limitations * under the License. */ + package org.elasticsearch.cloud.gce.tests; import org.junit.Test; diff --git a/src/test/java/org/elasticsearch/cloud/gce/tests/GceSameTagsTest.java b/src/test/java/org/elasticsearch/cloud/gce/tests/GceSameTagsTest.java index 4da09b2dd2e..a327e65a640 100644 --- a/src/test/java/org/elasticsearch/cloud/gce/tests/GceSameTagsTest.java +++ b/src/test/java/org/elasticsearch/cloud/gce/tests/GceSameTagsTest.java @@ -1,11 +1,11 @@ /* - * Licensed to ElasticSearch under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -16,6 +16,7 @@ * specific language governing permissions and limitations * under the License. */ + package org.elasticsearch.cloud.gce.tests; import org.junit.Test; From 431f704d6daebfef7dcfc1b3559ded54c1a19530 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Mon, 13 Jan 2014 16:07:27 +0100 Subject: [PATCH 27/96] Move tests to elasticsearch test framework Closes #8. --- pom.xml | 26 +++++-- .../cloud/gce/tests/GceAbstractTest.java | 73 +++---------------- 2 files changed, 29 insertions(+), 70 deletions(-) diff --git a/pom.xml b/pom.xml index d446ae1787a..6c5701b0d27 100644 --- a/pom.xml +++ b/pom.xml @@ -1,7 +1,7 @@ - - 0.90.3 + 0.90.10 v1beta15-rev3-1.15.0-rc 1.15.0-rc + 4.6.0 + 1 + true + onerror + + INFO + + org.apache.lucene + lucene-test-framework + ${lucene.version} + test + + org.elasticsearch @@ -90,9 +103,10 @@ governing permissions and limitations under the License. --> test - junit - junit - 4.11 + org.elasticsearch + elasticsearch + ${elasticsearch.version} + test-jar test diff --git a/src/test/java/org/elasticsearch/cloud/gce/tests/GceAbstractTest.java b/src/test/java/org/elasticsearch/cloud/gce/tests/GceAbstractTest.java index 05804ffdd73..8e41eb14be3 100644 --- a/src/test/java/org/elasticsearch/cloud/gce/tests/GceAbstractTest.java +++ b/src/test/java/org/elasticsearch/cloud/gce/tests/GceAbstractTest.java @@ -20,28 +20,14 @@ package org.elasticsearch.cloud.gce.tests; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; -import org.elasticsearch.client.Client; -import org.elasticsearch.client.transport.TransportClient; import org.elasticsearch.cloud.gce.GceComputeService; -import org.elasticsearch.common.io.FileSystemUtils; import org.elasticsearch.common.settings.ImmutableSettings; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.transport.TransportAddress; -import org.elasticsearch.node.Node; -import org.elasticsearch.node.NodeBuilder; -import org.elasticsearch.node.internal.InternalNode; -import org.elasticsearch.transport.netty.NettyTransport; -import org.junit.After; +import org.elasticsearch.test.ElasticsearchIntegrationTest; import org.junit.Assert; -import org.junit.Before; -import java.io.File; -import java.util.ArrayList; -import java.util.List; +@ElasticsearchIntegrationTest.ClusterScope(scope = ElasticsearchIntegrationTest.Scope.TEST, numNodes = 0) +public abstract class GceAbstractTest extends ElasticsearchIntegrationTest { -public abstract class GceAbstractTest { - - private static List nodes; private Class mock; public GceAbstractTest(Class mock) { @@ -49,63 +35,22 @@ public abstract class GceAbstractTest { this.mock = mock; } - @Before - public void setUp() { - nodes = new ArrayList(); - - File dataDir = new File("./target/es/data"); - if(dataDir.exists()) { - FileSystemUtils.deleteRecursively(dataDir, true); - } - } - - @After - public void tearDown() { - // Cleaning nodes after test - for (Node node : nodes) { - node.close(); - } - } - - protected Client getClient() { - // Create a TransportClient on node 1 and 2 - Settings settings = ImmutableSettings.settingsBuilder() - .put("cluster.name", "gce").build(); - - TransportClient client = new TransportClient(settings); - - for (Node node : nodes) { - NettyTransport nettyTransport = ((InternalNode) node).injector().getInstance(NettyTransport.class); - TransportAddress transportAddress = nettyTransport.boundAddress().publishAddress(); - client.addTransportAddress(transportAddress); - } - - return client; - } - protected void checkNumberOfNodes(int expected) { - NodesInfoResponse nodeInfos = getClient().admin().cluster().prepareNodesInfo().execute().actionGet(); + NodesInfoResponse nodeInfos = client().admin().cluster().prepareNodesInfo().execute().actionGet(); Assert.assertNotNull(nodeInfos.getNodes()); Assert.assertEquals(expected, nodeInfos.getNodes().length); } protected void nodeBuilder(String filteredTags) { - ImmutableSettings.Builder builder = ImmutableSettings.settingsBuilder() - //.put("gateway.type", "local") - .put("path.data", "./target/es/data") - .put("path.logs", "./target/es/logs") - .put("path.work", "./target/es/work") + ImmutableSettings.Builder nodeSettings = ImmutableSettings.settingsBuilder() .put("cloud.gce.api.impl", mock) - .put("cloud.gce.refresh_interval", "5s") - .put("node.name", (nodes.size()+1) + "#" + mock.getSimpleName()); + .put("cloud.gce.refresh_interval", "5s"); if (filteredTags != null) { - builder.put("discovery.gce.tags", filteredTags); + nodeSettings.put("discovery.gce.tags", filteredTags); } else { - builder.put("discovery.gce.tags", ""); + nodeSettings.put("discovery.gce.tags", ""); } - - Node node = NodeBuilder.nodeBuilder().settings(builder).node(); - nodes.add(node); + cluster().startNode(nodeSettings); } } From 853519ae9592e8f5bd4b3c92c8dc7523bae54af3 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Mon, 13 Jan 2014 16:12:52 +0100 Subject: [PATCH 28/96] Update to elasticsearch 1.0.0.RC1 Closes #7. --- pom.xml | 4 ++-- .../elasticsearch/cloud/gce/GceComputeServiceImpl.java | 8 ++++---- .../cloud/gce/tests/GceComputeServiceAbstractMock.java | 8 ++++---- src/test/resources/log4j.xml | 4 ++-- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/pom.xml b/pom.xml index 6c5701b0d27..617f8cf5cdf 100644 --- a/pom.xml +++ b/pom.xml @@ -17,7 +17,7 @@ governing permissions and limitations under the License. --> 4.0.0 org.elasticsearch elasticsearch-cloud-gce - 1.2.0-SNAPSHOT + 2.0.0.RC1-SNAPSHOT Google Compute Engine Cloud plugin for ElasticSearch 2013 @@ -47,7 +47,7 @@ governing permissions and limitations under the License. --> - 0.90.10 + 1.0.0.RC1-SNAPSHOT v1beta15-rev3-1.15.0-rc 1.15.0-rc 4.6.0 diff --git a/src/main/java/org/elasticsearch/cloud/gce/GceComputeServiceImpl.java b/src/main/java/org/elasticsearch/cloud/gce/GceComputeServiceImpl.java index 8ff193909d9..121fb5a368d 100644 --- a/src/main/java/org/elasticsearch/cloud/gce/GceComputeServiceImpl.java +++ b/src/main/java/org/elasticsearch/cloud/gce/GceComputeServiceImpl.java @@ -27,7 +27,7 @@ import com.google.api.client.json.jackson2.JacksonFactory; import com.google.api.services.compute.Compute; import com.google.api.services.compute.model.Instance; import com.google.api.services.compute.model.InstanceList; -import org.elasticsearch.ElasticSearchException; +import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.component.AbstractLifecycleComponent; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; @@ -115,14 +115,14 @@ public class GceComputeServiceImpl extends AbstractLifecycleComponent - + - + From b2247c977ab3f0b86ff97c22dcb72497683dbc38 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Wed, 15 Jan 2014 19:07:03 +0100 Subject: [PATCH 29/96] prepare release elasticsearch-cloud-gce-2.0.0.RC1 --- README.md | 5 +++-- pom.xml | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index d59630ca240..2a01ef985c3 100644 --- a/README.md +++ b/README.md @@ -3,11 +3,12 @@ Google Compute Engine Cloud Plugin for Elasticsearch The GCE Cloud plugin allows to use GCE API for the unicast discovery mechanism. -In order to install the plugin, simply run: `bin/plugin -install elasticsearch/elasticsearch-cloud-gce/1.1.0`. +In order to install the plugin, simply run: `bin/plugin -install elasticsearch/elasticsearch-cloud-gce/2.0.0.RC1`. | GCE Cloud Plugin | elasticsearch | Release date | |----------------------------|---------------------|:------------:| -| 2.0.0.RC1-SNAPSHOT (master)| 1.0.0.RC1 -> master | | +| 2.0.0-SNAPSHOT (master) | 1.0.0.RC1 -> master | | +| 2.0.0.RC1 | 1.0.0.RC1 -> master | 2014-01-15 | | 1.2.0-SNAPSHOT (1.x) | 0.90.3 -> 0.90 | | | 1.1.0 | 0.90.3 -> 0.90 | 2013-08-09 | | 1.0.0 | 0.90.0 -> 0.90.2 | 2013-07-23 | diff --git a/pom.xml b/pom.xml index 617f8cf5cdf..847862b202e 100644 --- a/pom.xml +++ b/pom.xml @@ -17,7 +17,7 @@ governing permissions and limitations under the License. --> 4.0.0 org.elasticsearch elasticsearch-cloud-gce - 2.0.0.RC1-SNAPSHOT + 2.0.0.RC1 Google Compute Engine Cloud plugin for ElasticSearch 2013 @@ -47,7 +47,7 @@ governing permissions and limitations under the License. --> - 1.0.0.RC1-SNAPSHOT + 1.0.0.RC1 v1beta15-rev3-1.15.0-rc 1.15.0-rc 4.6.0 From fd0354e26f4eb8ff021ae2880648afe5be34817c Mon Sep 17 00:00:00 2001 From: David Pilato Date: Wed, 15 Jan 2014 19:51:46 +0100 Subject: [PATCH 30/96] prepare for next development iteration --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 847862b202e..ad5e0022dbd 100644 --- a/pom.xml +++ b/pom.xml @@ -17,7 +17,7 @@ governing permissions and limitations under the License. --> 4.0.0 org.elasticsearch elasticsearch-cloud-gce - 2.0.0.RC1 + 2.0.0-SNAPSHOT Google Compute Engine Cloud plugin for ElasticSearch 2013 From b065179367e1805ba53bd045d5e13e4bd5eab733 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Tue, 28 Jan 2014 21:45:23 +0100 Subject: [PATCH 31/96] Add more traces when failing --- .../org/elasticsearch/cloud/gce/GceComputeServiceImpl.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/elasticsearch/cloud/gce/GceComputeServiceImpl.java b/src/main/java/org/elasticsearch/cloud/gce/GceComputeServiceImpl.java index 121fb5a368d..44f2cbc0c6b 100644 --- a/src/main/java/org/elasticsearch/cloud/gce/GceComputeServiceImpl.java +++ b/src/main/java/org/elasticsearch/cloud/gce/GceComputeServiceImpl.java @@ -51,12 +51,15 @@ public class GceComputeServiceImpl extends AbstractLifecycleComponent instances() { try { + logger.debug("get instances for project [{}], zone [{}]", project, zone); + Compute.Instances.List list = client().instances().list(project, zone); InstanceList instanceList = list.execute(); return instanceList.getItems(); } catch (IOException e) { - logger.warn("can not get list of nodes. Disabling GCE discovery."); + logger.warn("disabling GCE discovery. Can not get list of nodes: {}", e.getMessage()); + logger.debug("Full exception:", e); return new ArrayList(); } } From 4f11862cce4034f5322d348a595974dad96d6a0f Mon Sep 17 00:00:00 2001 From: David Pilato Date: Wed, 29 Jan 2014 11:21:45 +0100 Subject: [PATCH 32/96] Update to GCE API 1.17.0-rc Seen in #9, Google changed its APIs. We need to update as it previous version causes a `404` error. Closes #12. Closes #9. --- README.md | 38 ++++++++++--------- pom.xml | 16 +++++++- .../gce/GceUnicastHostsProvider.java | 5 +++ src/test/resources/log4j.xml | 2 +- 4 files changed, 40 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 2a01ef985c3..aa1bbcd4256 100644 --- a/README.md +++ b/README.md @@ -5,13 +5,13 @@ The GCE Cloud plugin allows to use GCE API for the unicast discovery mechanism. In order to install the plugin, simply run: `bin/plugin -install elasticsearch/elasticsearch-cloud-gce/2.0.0.RC1`. -| GCE Cloud Plugin | elasticsearch | Release date | -|----------------------------|---------------------|:------------:| -| 2.0.0-SNAPSHOT (master) | 1.0.0.RC1 -> master | | -| 2.0.0.RC1 | 1.0.0.RC1 -> master | 2014-01-15 | -| 1.2.0-SNAPSHOT (1.x) | 0.90.3 -> 0.90 | | -| 1.1.0 | 0.90.3 -> 0.90 | 2013-08-09 | -| 1.0.0 | 0.90.0 -> 0.90.2 | 2013-07-23 | +| GCE Cloud Plugin | elasticsearch | gce api | Release date | +|----------------------------|---------------------|--------------|:------------:| +| 2.0.0-SNAPSHOT (master) | 1.0.0.RC1 -> master | 1.17.0-rc | | +| 2.0.0.RC1 | 1.0.0.RC1 -> master | 1.15.0-rc | 2014-01-15 | +| 1.2.0-SNAPSHOT (1.x) | 0.90.3 -> 0.90 | 1.17.0-rc | | +| 1.1.0 | 0.90.3 -> 0.90 | 1.15.0-rc | 2013-08-09 | +| 1.0.0 | 0.90.0 -> 0.90.2 | 1.15.0-rc | 2013-07-23 | Google Compute Engine Virtual Machine Discovery @@ -98,13 +98,13 @@ sudo apt-get update sudo apt-get install curl # Download Elasticsearch -curl https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-0.90.2.deb -o elasticsearch-0.90.2.deb +wget https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-1.0.0.RC1.deb # Prepare Java installation sudo apt-get install java7-runtime-headless # Prepare Elasticsearch installation -sudo dpkg -i elasticsearch-0.90.2.deb +sudo dpkg -i elasticsearch-1.0.0.RC1.deb # Check that elasticsearch is running: curl http://localhost:9200/ @@ -116,10 +116,13 @@ This command should give you a JSON result: { "ok" : true, "status" : 200, - "name" : "Lunatica", + "name" : "Shriker", "version" : { - "number" : "0.90.2", - "snapshot_build" : false + "number" : "1.0.0.RC1", + "build_hash" : "0a5781f44876e8d1c30b6360628d59cb2a7a2bbb", + "build_timestamp" : "2014-01-10T10:18:37Z", + "build_snapshot" : false, + "lucene_version" : "4.6" }, "tagline" : "You Know, for Search" } @@ -131,8 +134,7 @@ Install the plugin: ```sh # Use Plugin Manager to install it -sudo /usr/share/elasticsearch/bin/plugin - --install elasticsearch/elasticsearch-cloud-gce/1.1.0 +sudo /usr/share/elasticsearch/bin/plugin --install elasticsearch/elasticsearch-cloud-gce/2.0.0.RC1 # Configure it: sudo vi /etc/elasticsearch/elasticsearch.yml @@ -206,7 +208,7 @@ gcutil listkernels --project es-cloud | projects/google/global/kernels/gce-v20130603 | SCSI-enabled 3.3.8-gcg built 2013-05-29 01:04:00 | | +----------------------------------------------+--------------------------------------------------+-------------+ # Note the kernel you prefer to use and add your image to your catalog: -gcutil --project=es-cloud addimage elasticsearch-0-90-2 \ +gcutil --project=es-cloud addimage elasticsearch-1-0-0-RC1 \ gs://esimage/e4686d7f5bf904a924ae0cfeb58d0827c6d5b966.image.tar.gz \ --preferred_kernel=projects/google/global/kernels/gce-v20130603 @@ -220,7 +222,7 @@ As you have now an image, you can create as many instances as you need: ```sh # Just change node name (here myesnode2) -gcutil --project=es-cloud addinstance --image=elasticsearch-0-90-2 \ +gcutil --project=es-cloud addinstance --image=elasticsearch-1-0-0-RC1 \ --kernel=projects/google/global/kernels/gce-v20130603 myesnode2 \ --zone europe-west1-a --machine_type f1-micro --service_account_scope=compute-rw \ --persistent_boot_disk @@ -293,7 +295,7 @@ gcutil --project=es-cloud addinstance myesnode1 \ --metadata=es_port:9301 # when creating an instance from an image -gcutil --project=es-cloud addinstance --image=elasticsearch-0-90-2 \ +gcutil --project=es-cloud addinstance --image=elasticsearch-1-0-0-RC1 \ --kernel=projects/google/global/kernels/gce-v20130603 myesnode2 \ --zone europe-west1-a --machine_type f1-micro --service_account_scope=compute-rw \ --persistent_boot_disk --metadata=es_port:9301 @@ -340,7 +342,7 @@ License This software is licensed under the Apache 2 license, quoted below. - Copyright 2009-2013 ElasticSearch + Copyright 2009-2014 Elasticsearch Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of diff --git a/pom.xml b/pom.xml index ad5e0022dbd..757cf322b86 100644 --- a/pom.xml +++ b/pom.xml @@ -48,8 +48,8 @@ governing permissions and limitations under the License. --> 1.0.0.RC1 - v1beta15-rev3-1.15.0-rc - 1.15.0-rc + v1beta16-rev1-1.17.0-rc + 1.17.0-rc 4.6.0 1 true @@ -109,6 +109,18 @@ governing permissions and limitations under the License. --> test-jar test + + org.hamcrest + hamcrest-core + 1.3.RC2 + test + + + org.hamcrest + hamcrest-library + 1.3.RC2 + test + diff --git a/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java b/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java index 15c498a752b..06ba581c0df 100644 --- a/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java +++ b/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java @@ -123,6 +123,11 @@ public class GceUnicastHostsProvider extends AbstractComponent implements Unicas try { Collection instances = gceComputeService.instances(); + if (instances == null) { + logger.trace("no instance found for project [{}], zone [{}].", this.project, this.zone); + return cachedDiscoNodes; + } + for (Instance instance : instances) { String name = instance.getName(); String type = instance.getMachineType(); diff --git a/src/test/resources/log4j.xml b/src/test/resources/log4j.xml index 8e7ddf7591d..07be97b44d7 100644 --- a/src/test/resources/log4j.xml +++ b/src/test/resources/log4j.xml @@ -1,5 +1,5 @@ - - GCE cloud plugin 4.0.0 org.elasticsearch elasticsearch-cloud-gce 2.0.0-SNAPSHOT - Google Compute Engine Cloud plugin for ElasticSearch + Elasticsearch Google Compute Engine cloud plugin + The Google Compute Engine (GCE) Cloud plugin allows to use GCE API for the unicast discovery mechanism. + https://github.com/elasticsearch/elasticsearch-cloud-gce/ 2013 From f39ba93ca8e458de0be02e95f4188d422176c0f3 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Tue, 11 Mar 2014 17:45:58 +0100 Subject: [PATCH 35/96] Update to elasticsearch 1.0.0 Closes #14. --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 699963dd7dc..4611a4ac52d 100644 --- a/pom.xml +++ b/pom.xml @@ -48,10 +48,10 @@ governing permissions and limitations under the License. --> - 1.0.0.RC1 + 1.0.1 v1beta16-rev1-1.17.0-rc 1.17.0-rc - 4.6.0 + 4.6.1 1 true onerror From 6aa36aa5be3e3ca56ee48a84f18519ed9e00f5cc Mon Sep 17 00:00:00 2001 From: David Pilato Date: Wed, 12 Mar 2014 12:03:27 +0100 Subject: [PATCH 36/96] Move tests to elasticsearch test framework Closes #16. --- .gitignore | 1 + pom.xml | 95 +++++++++++++++++++ .../gce/GceUnicastHostsProvider.java | 8 +- .../cloud/gce/AbstractGceTest.java | 61 ++++++++++++ .../cloud/gce/tests/GceAbstractTest.java | 56 ----------- .../cloud/gce/tests/GceDifferentTagsTest.java | 65 ------------- .../cloud/gce/tests/GceSameTagsTest.java | 65 ------------- .../gce/AbstractGceComputeServiceTest.java | 68 +++++++++++++ .../gce}/GceComputeServiceAbstractMock.java | 23 +++-- ...mputeServiceTwoNodesDifferentTagsMock.java | 2 +- ...GceComputeServiceTwoNodesSameTagsMock.java | 3 +- .../gce/GceDifferentTagsNoTagTest.java | 42 ++++++++ .../gce/GceDifferentTagsOneTagTest.java | 43 +++++++++ .../gce/GceDifferentTagsTwoTagsTest.java | 43 +++++++++ .../discovery/gce/GceSameTagsNoTagTest.java | 42 ++++++++ .../discovery/gce/GceSameTagsOneTagTest.java | 43 +++++++++ .../discovery/gce/GceSameTagsTwoTagsTest.java | 43 +++++++++ .../discovery/gce/PropertiesHelper.java | 34 +++++++ .../gce/itest/GceSimpleITest.java | 60 ++++++++++++ src/test/resources/elasticsearch.yml | 28 ++---- src/test/resources/log4j.xml | 6 +- src/test/resources/plugin-test.properties | 3 + 22 files changed, 610 insertions(+), 224 deletions(-) create mode 100644 src/test/java/org/elasticsearch/cloud/gce/AbstractGceTest.java delete mode 100644 src/test/java/org/elasticsearch/cloud/gce/tests/GceAbstractTest.java delete mode 100644 src/test/java/org/elasticsearch/cloud/gce/tests/GceDifferentTagsTest.java delete mode 100644 src/test/java/org/elasticsearch/cloud/gce/tests/GceSameTagsTest.java create mode 100644 src/test/java/org/elasticsearch/discovery/gce/AbstractGceComputeServiceTest.java rename src/test/java/org/elasticsearch/{cloud/gce/tests => discovery/gce}/GceComputeServiceAbstractMock.java (86%) rename src/test/java/org/elasticsearch/{cloud/gce/tests => discovery/gce}/GceComputeServiceTwoNodesDifferentTagsMock.java (97%) rename src/test/java/org/elasticsearch/{cloud/gce/tests => discovery/gce}/GceComputeServiceTwoNodesSameTagsMock.java (93%) create mode 100644 src/test/java/org/elasticsearch/discovery/gce/GceDifferentTagsNoTagTest.java create mode 100644 src/test/java/org/elasticsearch/discovery/gce/GceDifferentTagsOneTagTest.java create mode 100644 src/test/java/org/elasticsearch/discovery/gce/GceDifferentTagsTwoTagsTest.java create mode 100644 src/test/java/org/elasticsearch/discovery/gce/GceSameTagsNoTagTest.java create mode 100644 src/test/java/org/elasticsearch/discovery/gce/GceSameTagsOneTagTest.java create mode 100644 src/test/java/org/elasticsearch/discovery/gce/GceSameTagsTwoTagsTest.java create mode 100644 src/test/java/org/elasticsearch/discovery/gce/PropertiesHelper.java create mode 100644 src/test/java/org/elasticsearch/gce/itest/GceSimpleITest.java create mode 100644 src/test/resources/plugin-test.properties diff --git a/.gitignore b/.gitignore index 9dac7bc0ead..87879ffa87d 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ /.project /.classpath *.vmoptions +.local-execution-hints.log diff --git a/pom.xml b/pom.xml index 4611a4ac52d..e4850e88db8 100644 --- a/pom.xml +++ b/pom.xml @@ -56,6 +56,7 @@ governing permissions and limitations under the License. --> true onerror + 9300 INFO @@ -145,6 +146,7 @@ governing permissions and limitations under the License. --> src/test/resources true + plugin-test.properties elasticsearch.yml @@ -166,10 +168,103 @@ governing permissions and limitations under the License. --> 1.6 + + com.carrotsearch.randomizedtesting + junit4-maven-plugin + 2.0.14 + + + tests + test + + junit4 + + + 20 + pipe,warn + true + + + + + + + + + ${tests.jvms} + + + + + + + **/*Tests.class + **/*Test.class + + + **/Abstract*.class + **/*StressTest.class + + + ${tests.jvm.argline} + + + -Xmx512m + -Xss256k + -XX:MaxDirectMemorySize=512m + -Des.logger.prefix= + + ${tests.shuffle} + ${tests.verbose} + ${tests.seed} + ${tests.failfast} + + + ${tests.jvm.argline} + ${tests.iters} + ${tests.maxfailures} + ${tests.failfast} + ${tests.class} + ${tests.method} + ${tests.nightly} + ${tests.badapples} + ${tests.weekly} + ${tests.slow} + ${tests.gce} + ${tests.awaitsfix} + ${tests.slow} + ${tests.timeoutSuite} + ${tests.showSuccess} + ${tests.integration} + ${tests.cluster_seed} + ${tests.client.ratio} + false + network + ${es.config} + ${es.logger.level} + true + + + + + org.apache.maven.plugins maven-surefire-plugin 2.15 + + true + org.apache.maven.plugins diff --git a/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java b/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java index 06ba581c0df..a0238b00b4f 100644 --- a/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java +++ b/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java @@ -34,7 +34,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.discovery.zen.ping.unicast.UnicastHostsProvider; -import org.elasticsearch.discovery.zen.ping.unicast.UnicastZenPing; import org.elasticsearch.transport.TransportService; import java.io.IOException; @@ -225,8 +224,9 @@ public class GceUnicastHostsProvider extends AbstractComponent implements Unicas // ip_private is a single IP Address. We need to build a TransportAddress from it TransportAddress[] addresses = transportService.addressesFromString(address); + // If user has set `es_port` metadata, we don't need to ping all ports // we only limit to 1 addresses, makes no sense to ping 100 ports - for (int i = 0; (i < addresses.length && i < UnicastZenPing.LIMIT_PORTS_COUNT); i++) { + for (int i = 0; i < addresses.length; i++) { logger.trace("adding {}, type {}, image {}, address {}, transport_address {}, status {}", name, type, image, ip_private, addresses[i], status); cachedDiscoNodes.add(new DiscoveryNode("#cloud-" + name + "-" + i, addresses[i], Version.CURRENT)); @@ -248,11 +248,9 @@ public class GceUnicastHostsProvider extends AbstractComponent implements Unicas return cachedDiscoNodes; } - private boolean checkProperty(String name, String value) { + private void checkProperty(String name, String value) { if (!Strings.hasText(value)) { logger.warn("cloud.gce.{} is not set.", name); - return false; } - return true; } } diff --git a/src/test/java/org/elasticsearch/cloud/gce/AbstractGceTest.java b/src/test/java/org/elasticsearch/cloud/gce/AbstractGceTest.java new file mode 100644 index 00000000000..115775f7c6d --- /dev/null +++ b/src/test/java/org/elasticsearch/cloud/gce/AbstractGceTest.java @@ -0,0 +1,61 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.cloud.gce; + +import com.carrotsearch.randomizedtesting.annotations.TestGroup; +import org.elasticsearch.test.ElasticsearchIntegrationTest; + +import java.lang.annotation.Documented; +import java.lang.annotation.Inherited; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +/** + * + */ +public abstract class AbstractGceTest extends ElasticsearchIntegrationTest { + + /** + * Annotation for tests that require GCE to run. GCE tests are disabled by default. + *

+ * To enable test add -Dtests.gce=true -Des.config=/path/to/elasticsearch.yml + *

+ * The elasticsearch.yml file should contain the following keys + * 

+      cloud:
+          gce:
+              project_id: es-cloud
+              zone: europe-west1-a
+      discovery:
+              type: gce
+     *
+ */ + @Documented + @Inherited + @Retention(RetentionPolicy.RUNTIME) + @TestGroup(enabled = false, sysProperty = SYSPROP_GCE) + public @interface GceTest { + } + + /** + */ + public static final String SYSPROP_GCE = "tests.gce"; + +} diff --git a/src/test/java/org/elasticsearch/cloud/gce/tests/GceAbstractTest.java b/src/test/java/org/elasticsearch/cloud/gce/tests/GceAbstractTest.java deleted file mode 100644 index 8e41eb14be3..00000000000 --- a/src/test/java/org/elasticsearch/cloud/gce/tests/GceAbstractTest.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.cloud.gce.tests; - -import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; -import org.elasticsearch.cloud.gce.GceComputeService; -import org.elasticsearch.common.settings.ImmutableSettings; -import org.elasticsearch.test.ElasticsearchIntegrationTest; -import org.junit.Assert; - -@ElasticsearchIntegrationTest.ClusterScope(scope = ElasticsearchIntegrationTest.Scope.TEST, numNodes = 0) -public abstract class GceAbstractTest extends ElasticsearchIntegrationTest { - - private Class mock; - - public GceAbstractTest(Class mock) { - // We want to inject the GCE API Mock - this.mock = mock; - } - - protected void checkNumberOfNodes(int expected) { - NodesInfoResponse nodeInfos = client().admin().cluster().prepareNodesInfo().execute().actionGet(); - - Assert.assertNotNull(nodeInfos.getNodes()); - Assert.assertEquals(expected, nodeInfos.getNodes().length); - } - - protected void nodeBuilder(String filteredTags) { - ImmutableSettings.Builder nodeSettings = ImmutableSettings.settingsBuilder() - .put("cloud.gce.api.impl", mock) - .put("cloud.gce.refresh_interval", "5s"); - if (filteredTags != null) { - nodeSettings.put("discovery.gce.tags", filteredTags); - } else { - nodeSettings.put("discovery.gce.tags", ""); - } - cluster().startNode(nodeSettings); - } -} diff --git a/src/test/java/org/elasticsearch/cloud/gce/tests/GceDifferentTagsTest.java b/src/test/java/org/elasticsearch/cloud/gce/tests/GceDifferentTagsTest.java deleted file mode 100644 index 155f99ecaf4..00000000000 --- a/src/test/java/org/elasticsearch/cloud/gce/tests/GceDifferentTagsTest.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.cloud.gce.tests; - -import org.junit.Test; - -public class GceDifferentTagsTest extends GceAbstractTest { - - public GceDifferentTagsTest() { - super(GceComputeServiceTwoNodesDifferentTagsMock.class); - } - - @Test - public void two_nodes_should_run_no_tag() { - // Then we start our node for tests - nodeBuilder(null); - - // Let's start a second node - nodeBuilder(null); - - // We expect having 1 nodes as part of the cluster, let's test that - checkNumberOfNodes(2); - } - - @Test - public void one_node_should_run_two_tags() { - // Then we start our node for tests - nodeBuilder("elasticsearch,dev"); - - // Let's start a second node - nodeBuilder("elasticsearch,dev"); - - // We expect having 1 node as part of the cluster, let's test that - checkNumberOfNodes(1); - } - - @Test - public void one_node_should_be_filtered_one_tag() { - // Then we start our node for tests - nodeBuilder("elasticsearch"); - - // Let's start a second node - nodeBuilder("elasticsearch"); - - // We expect having 1 nodes as part of the cluster, let's test that - checkNumberOfNodes(1); - } -} diff --git a/src/test/java/org/elasticsearch/cloud/gce/tests/GceSameTagsTest.java b/src/test/java/org/elasticsearch/cloud/gce/tests/GceSameTagsTest.java deleted file mode 100644 index a327e65a640..00000000000 --- a/src/test/java/org/elasticsearch/cloud/gce/tests/GceSameTagsTest.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.cloud.gce.tests; - -import org.junit.Test; - -public class GceSameTagsTest extends GceAbstractTest { - - public GceSameTagsTest() { - super(GceComputeServiceTwoNodesSameTagsMock.class); - } - - @Test - public void two_nodes_should_run_no_tag() { - // Then we start our node for tests - nodeBuilder(null); - - // Let's start a second node - nodeBuilder(null); - - // We expect having 1 nodes as part of the cluster, let's test that - checkNumberOfNodes(2); - } - - @Test - public void two_nodes_should_run_two_tags() { - // Then we start our node for tests - nodeBuilder("elasticsearch,dev"); - - // Let's start a second node - nodeBuilder("elasticsearch,dev"); - - // We expect having 2 nodes as part of the cluster, let's test that - checkNumberOfNodes(2); - } - - @Test - public void two_nodes_should_run_one_tag() { - // Then we start our node for tests - nodeBuilder("elasticsearch"); - - // Let's start a second node - nodeBuilder("elasticsearch"); - - // We expect having 2 nodes as part of the cluster, let's test that - checkNumberOfNodes(2); - } -} diff --git a/src/test/java/org/elasticsearch/discovery/gce/AbstractGceComputeServiceTest.java b/src/test/java/org/elasticsearch/discovery/gce/AbstractGceComputeServiceTest.java new file mode 100644 index 00000000000..a7c4220781b --- /dev/null +++ b/src/test/java/org/elasticsearch/discovery/gce/AbstractGceComputeServiceTest.java @@ -0,0 +1,68 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.discovery.gce; + +import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; +import org.elasticsearch.common.settings.ImmutableSettings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.test.ElasticsearchIntegrationTest; +import org.junit.Test; + +import java.io.IOException; + +import static org.hamcrest.CoreMatchers.notNullValue; +import static org.hamcrest.Matchers.is; + +@ElasticsearchIntegrationTest.ClusterScope( + scope = ElasticsearchIntegrationTest.Scope.SUITE, + numNodes = 2, + transportClientRatio = 0.0) +public abstract class AbstractGceComputeServiceTest extends ElasticsearchIntegrationTest { + /** + * Set the number of expected nodes in the current cluster + */ + protected abstract int getExpectedNodes(); + + public static int getPort(int nodeOrdinal) { + try { + return PropertiesHelper.getAsInt("plugin.port") + + nodeOrdinal * 10; + } catch (IOException e) { + } + + return -1; + } + + @Override + protected Settings nodeSettings(int nodeOrdinal) { + return ImmutableSettings.builder() + .put("transport.tcp.port", getPort(nodeOrdinal)) + .put("http.enabled", false) + .build(); + } + + @Test + public void testExpectedNumberOfNodes() { + NodesInfoResponse nodeInfos = client().admin().cluster().prepareNodesInfo().execute().actionGet(); + + assertThat(nodeInfos.getNodes(), notNullValue()); + assertThat(nodeInfos.getNodes().length, is(getExpectedNodes())); + } +} diff --git a/src/test/java/org/elasticsearch/cloud/gce/tests/GceComputeServiceAbstractMock.java b/src/test/java/org/elasticsearch/discovery/gce/GceComputeServiceAbstractMock.java similarity index 86% rename from src/test/java/org/elasticsearch/cloud/gce/tests/GceComputeServiceAbstractMock.java rename to src/test/java/org/elasticsearch/discovery/gce/GceComputeServiceAbstractMock.java index 692939af4f1..4a13466b0c6 100644 --- a/src/test/java/org/elasticsearch/cloud/gce/tests/GceComputeServiceAbstractMock.java +++ b/src/test/java/org/elasticsearch/discovery/gce/GceComputeServiceAbstractMock.java @@ -17,7 +17,7 @@ * under the License. */ -package org.elasticsearch.cloud.gce.tests; +package org.elasticsearch.discovery.gce; import com.google.api.services.compute.model.Instance; import com.google.api.services.compute.model.Metadata; @@ -48,13 +48,15 @@ public abstract class GceComputeServiceAbstractMock extends AbstractLifecycleCom } } - @Override - public Collection instances() { - Collection instances = new ArrayList(); + private Collection instances = null; + private void computeInstances() { + instances = new ArrayList(); + + int nodeNumber = 0; // For each instance (item of tags) - int port = 9300; for (List tags : getTags()) { + logger.info(" ----> GCE Mock API: Adding node {}", nodeNumber); Instance instance = new Instance(); instance.setName("Mock Node " + tags); instance.setMachineType("Mock Type machine"); @@ -71,13 +73,20 @@ public abstract class GceComputeServiceAbstractMock extends AbstractLifecycleCom // Add metadata es_port:930X where X is the instance number Metadata metadata = new Metadata(); - metadata.put("es_port", "" + port); + metadata.put("es_port", "" + AbstractGceComputeServiceTest.getPort(nodeNumber)); instance.setMetadata(metadata); + instances.add(instance); - port++; + nodeNumber++; } + } + @Override + public Collection instances() { + if (instances == null || instances.size() == 0) { + computeInstances(); + } return instances; } diff --git a/src/test/java/org/elasticsearch/cloud/gce/tests/GceComputeServiceTwoNodesDifferentTagsMock.java b/src/test/java/org/elasticsearch/discovery/gce/GceComputeServiceTwoNodesDifferentTagsMock.java similarity index 97% rename from src/test/java/org/elasticsearch/cloud/gce/tests/GceComputeServiceTwoNodesDifferentTagsMock.java rename to src/test/java/org/elasticsearch/discovery/gce/GceComputeServiceTwoNodesDifferentTagsMock.java index 5981342909c..57878deb4ea 100644 --- a/src/test/java/org/elasticsearch/cloud/gce/tests/GceComputeServiceTwoNodesDifferentTagsMock.java +++ b/src/test/java/org/elasticsearch/discovery/gce/GceComputeServiceTwoNodesDifferentTagsMock.java @@ -17,7 +17,7 @@ * under the License. */ -package org.elasticsearch.cloud.gce.tests; +package org.elasticsearch.discovery.gce; import org.elasticsearch.common.collect.Lists; import org.elasticsearch.common.inject.Inject; diff --git a/src/test/java/org/elasticsearch/cloud/gce/tests/GceComputeServiceTwoNodesSameTagsMock.java b/src/test/java/org/elasticsearch/discovery/gce/GceComputeServiceTwoNodesSameTagsMock.java similarity index 93% rename from src/test/java/org/elasticsearch/cloud/gce/tests/GceComputeServiceTwoNodesSameTagsMock.java rename to src/test/java/org/elasticsearch/discovery/gce/GceComputeServiceTwoNodesSameTagsMock.java index 0dea3b9eb2f..7b5be3687c5 100644 --- a/src/test/java/org/elasticsearch/cloud/gce/tests/GceComputeServiceTwoNodesSameTagsMock.java +++ b/src/test/java/org/elasticsearch/discovery/gce/GceComputeServiceTwoNodesSameTagsMock.java @@ -17,7 +17,7 @@ * under the License. */ -package org.elasticsearch.cloud.gce.tests; +package org.elasticsearch.discovery.gce; import org.elasticsearch.common.collect.Lists; import org.elasticsearch.common.inject.Inject; @@ -43,6 +43,5 @@ public class GceComputeServiceTwoNodesSameTagsMock extends GceComputeServiceAbst @Inject protected GceComputeServiceTwoNodesSameTagsMock(Settings settings) { super(settings); - logger.debug("Starting Gce Mock {}", tags); } } diff --git a/src/test/java/org/elasticsearch/discovery/gce/GceDifferentTagsNoTagTest.java b/src/test/java/org/elasticsearch/discovery/gce/GceDifferentTagsNoTagTest.java new file mode 100644 index 00000000000..481adb8ebcc --- /dev/null +++ b/src/test/java/org/elasticsearch/discovery/gce/GceDifferentTagsNoTagTest.java @@ -0,0 +1,42 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.discovery.gce; + +import org.elasticsearch.common.settings.ImmutableSettings; +import org.elasticsearch.common.settings.Settings; + +public class GceDifferentTagsNoTagTest extends AbstractGceComputeServiceTest { + + @Override + protected Settings nodeSettings(int nodeOrdinal) { + ImmutableSettings.Builder settings = ImmutableSettings.settingsBuilder().put(super.nodeSettings(nodeOrdinal)); + settings.put("discovery.type", "gce"); + settings.put("cloud.gce.api.impl", GceComputeServiceTwoNodesDifferentTagsMock.class); + return settings.build(); + } + + /** + * Set the number of expected nodes in the current cluster + */ + @Override + protected int getExpectedNodes() { + return 2; + } +} diff --git a/src/test/java/org/elasticsearch/discovery/gce/GceDifferentTagsOneTagTest.java b/src/test/java/org/elasticsearch/discovery/gce/GceDifferentTagsOneTagTest.java new file mode 100644 index 00000000000..8ccb98f6fd7 --- /dev/null +++ b/src/test/java/org/elasticsearch/discovery/gce/GceDifferentTagsOneTagTest.java @@ -0,0 +1,43 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.discovery.gce; + +import org.elasticsearch.common.settings.ImmutableSettings; +import org.elasticsearch.common.settings.Settings; + +public class GceDifferentTagsOneTagTest extends AbstractGceComputeServiceTest { + + @Override + protected Settings nodeSettings(int nodeOrdinal) { + ImmutableSettings.Builder settings = ImmutableSettings.settingsBuilder().put(super.nodeSettings(nodeOrdinal)); + settings.put("discovery.type", "gce"); + settings.put("cloud.gce.api.impl", GceComputeServiceTwoNodesDifferentTagsMock.class); + settings.put("discovery.gce.tags", "elasticsearch"); + return settings.build(); + } + + /** + * Set the number of expected nodes in the current cluster + */ + @Override + protected int getExpectedNodes() { + return 1; + } +} diff --git a/src/test/java/org/elasticsearch/discovery/gce/GceDifferentTagsTwoTagsTest.java b/src/test/java/org/elasticsearch/discovery/gce/GceDifferentTagsTwoTagsTest.java new file mode 100644 index 00000000000..d414b02409f --- /dev/null +++ b/src/test/java/org/elasticsearch/discovery/gce/GceDifferentTagsTwoTagsTest.java @@ -0,0 +1,43 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.discovery.gce; + +import org.elasticsearch.common.settings.ImmutableSettings; +import org.elasticsearch.common.settings.Settings; + +public class GceDifferentTagsTwoTagsTest extends AbstractGceComputeServiceTest { + + @Override + protected Settings nodeSettings(int nodeOrdinal) { + ImmutableSettings.Builder settings = ImmutableSettings.settingsBuilder().put(super.nodeSettings(nodeOrdinal)); + settings.put("discovery.type", "gce"); + settings.put("cloud.gce.api.impl", GceComputeServiceTwoNodesDifferentTagsMock.class); + settings.put("discovery.gce.tags", "elasticsearch,dev"); + return settings.build(); + } + + /** + * Set the number of expected nodes in the current cluster + */ + @Override + protected int getExpectedNodes() { + return 1; + } +} diff --git a/src/test/java/org/elasticsearch/discovery/gce/GceSameTagsNoTagTest.java b/src/test/java/org/elasticsearch/discovery/gce/GceSameTagsNoTagTest.java new file mode 100644 index 00000000000..055816475c9 --- /dev/null +++ b/src/test/java/org/elasticsearch/discovery/gce/GceSameTagsNoTagTest.java @@ -0,0 +1,42 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.discovery.gce; + +import org.elasticsearch.common.settings.ImmutableSettings; +import org.elasticsearch.common.settings.Settings; + +public class GceSameTagsNoTagTest extends AbstractGceComputeServiceTest { + + @Override + protected Settings nodeSettings(int nodeOrdinal) { + ImmutableSettings.Builder settings = ImmutableSettings.settingsBuilder().put(super.nodeSettings(nodeOrdinal)); + settings.put("discovery.type", "gce"); + settings.put("cloud.gce.api.impl", GceComputeServiceTwoNodesSameTagsMock.class); + return settings.build(); + } + + /** + * Set the number of expected nodes in the current cluster + */ + @Override + protected int getExpectedNodes() { + return 2; + } +} diff --git a/src/test/java/org/elasticsearch/discovery/gce/GceSameTagsOneTagTest.java b/src/test/java/org/elasticsearch/discovery/gce/GceSameTagsOneTagTest.java new file mode 100644 index 00000000000..7dbd2339d11 --- /dev/null +++ b/src/test/java/org/elasticsearch/discovery/gce/GceSameTagsOneTagTest.java @@ -0,0 +1,43 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.discovery.gce; + +import org.elasticsearch.common.settings.ImmutableSettings; +import org.elasticsearch.common.settings.Settings; + +public class GceSameTagsOneTagTest extends AbstractGceComputeServiceTest { + + @Override + protected Settings nodeSettings(int nodeOrdinal) { + ImmutableSettings.Builder settings = ImmutableSettings.settingsBuilder().put(super.nodeSettings(nodeOrdinal)); + settings.put("discovery.type", "gce"); + settings.put("cloud.gce.api.impl", GceComputeServiceTwoNodesSameTagsMock.class); + settings.put("discovery.gce.tags", "elasticsearch"); + return settings.build(); + } + + /** + * Set the number of expected nodes in the current cluster + */ + @Override + protected int getExpectedNodes() { + return 2; + } +} diff --git a/src/test/java/org/elasticsearch/discovery/gce/GceSameTagsTwoTagsTest.java b/src/test/java/org/elasticsearch/discovery/gce/GceSameTagsTwoTagsTest.java new file mode 100644 index 00000000000..f2417a83657 --- /dev/null +++ b/src/test/java/org/elasticsearch/discovery/gce/GceSameTagsTwoTagsTest.java @@ -0,0 +1,43 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.discovery.gce; + +import org.elasticsearch.common.settings.ImmutableSettings; +import org.elasticsearch.common.settings.Settings; + +public class GceSameTagsTwoTagsTest extends AbstractGceComputeServiceTest { + + @Override + protected Settings nodeSettings(int nodeOrdinal) { + ImmutableSettings.Builder settings = ImmutableSettings.settingsBuilder().put(super.nodeSettings(nodeOrdinal)); + settings.put("discovery.type", "gce"); + settings.put("cloud.gce.api.impl", GceComputeServiceTwoNodesSameTagsMock.class); + settings.put("discovery.gce.tags", "elasticsearch,dev"); + return settings.build(); + } + + /** + * Set the number of expected nodes in the current cluster + */ + @Override + protected int getExpectedNodes() { + return 2; + } +} diff --git a/src/test/java/org/elasticsearch/discovery/gce/PropertiesHelper.java b/src/test/java/org/elasticsearch/discovery/gce/PropertiesHelper.java new file mode 100644 index 00000000000..6330135f3d3 --- /dev/null +++ b/src/test/java/org/elasticsearch/discovery/gce/PropertiesHelper.java @@ -0,0 +1,34 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.discovery.gce; + +import java.io.IOException; +import java.util.Properties; + +/** + * Get information from plugin-test.properties file + */ +public class PropertiesHelper { + public static int getAsInt(String name) throws IOException { + final Properties properties = new Properties(); + properties.load(PropertiesHelper.class.getResourceAsStream("/plugin-test.properties")); + return Integer.parseInt(properties.getProperty(name)); + } +} diff --git a/src/test/java/org/elasticsearch/gce/itest/GceSimpleITest.java b/src/test/java/org/elasticsearch/gce/itest/GceSimpleITest.java new file mode 100644 index 00000000000..5e2accbe827 --- /dev/null +++ b/src/test/java/org/elasticsearch/gce/itest/GceSimpleITest.java @@ -0,0 +1,60 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.gce.itest; + +import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; +import org.elasticsearch.cloud.gce.AbstractGceTest; +import org.elasticsearch.common.settings.ImmutableSettings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.test.ElasticsearchIntegrationTest; +import org.hamcrest.Matchers; +import org.junit.Test; + +/** + * This test needs GCE to run and -Dtests.gce=true to be set + * and -Des.config=/path/to/elasticsearch.yml + * TODO: By now, it will only work from GCE platform as we don't support yet external auth. + * See https://github.com/elasticsearch/elasticsearch-cloud-gce/issues/10 + * @see org.elasticsearch.cloud.gce.AbstractGceTest + */ +@AbstractGceTest.GceTest +@ElasticsearchIntegrationTest.ClusterScope( + scope = ElasticsearchIntegrationTest.Scope.TEST, + numNodes = 1, + transportClientRatio = 0.0) +public class GceSimpleITest extends AbstractGceTest { + + @Test + public void one_node_should_run() { + // Do nothing... Just start :-) + // but let's check that we have at least 1 node (local node) + ClusterStateResponse clusterState = client().admin().cluster().prepareState().execute().actionGet(); + + assertThat(clusterState.getState().getNodes().getSize(), Matchers.greaterThanOrEqualTo(1)); + } + + @Override + public Settings indexSettings() { + // During restore we frequently restore index to exactly the same state it was before, that might cause the same + // checksum file to be written twice during restore operation + return ImmutableSettings.builder().put(super.indexSettings()) + .build(); + } +} diff --git a/src/test/resources/elasticsearch.yml b/src/test/resources/elasticsearch.yml index 1d13d3e65b9..7fde927a0b6 100644 --- a/src/test/resources/elasticsearch.yml +++ b/src/test/resources/elasticsearch.yml @@ -9,7 +9,6 @@ # KIND, either express or implied. See the License for the specific language # governing permissions and limitations under the License. - cluster.name: gce # GCE discovery allows to use GCE API in order to perform discovery. # # You have to install the cloud-gce plugin for enabling the GCE discovery. @@ -18,22 +17,11 @@ # for more information. # # See README in elasticsearch-cloud-gce repository for a step-by-step tutorial. - cloud: - gce: - project_id: es-cloud - zone: europe-west1-a - discovery: - type: gce - gce: - tags: you_can_add_tags_here - -# Note that with tests, discovery.gce.tags will be replaced by test values - -# We set path here to use maven target dir - path: - data: ${project.build.directory}/es/data - logs: ${project.build.directory}/es/logs - work: ${project.build.directory}/es/work - -# We don't need http layer for tests - http.enabled: false +# cloud: +# gce: +# project_id: es-cloud +# zone: europe-west1-a +# discovery: +# type: gce +# gce: +# tags: you_can_add_tags_here diff --git a/src/test/resources/log4j.xml b/src/test/resources/log4j.xml index 07be97b44d7..7804dbaf027 100644 --- a/src/test/resources/log4j.xml +++ b/src/test/resources/log4j.xml @@ -21,14 +21,14 @@ governing permissions and limitations under the License. --> - + - + - + diff --git a/src/test/resources/plugin-test.properties b/src/test/resources/plugin-test.properties new file mode 100644 index 00000000000..a59dab1836c --- /dev/null +++ b/src/test/resources/plugin-test.properties @@ -0,0 +1,3 @@ +# This file contains specific needed properties for tests. +# You can use placeholders to extract values from Maven for example +plugin.port=${es.plugin.port} From cba743df3f807fed211151dd9258c2a95a88d218 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Wed, 12 Mar 2014 12:14:09 +0100 Subject: [PATCH 37/96] prepare release elasticsearch-cloud-gce-2.0.0 --- README.md | 4 ++-- pom.xml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 2fda00277c6..bf52a02cf7b 100644 --- a/README.md +++ b/README.md @@ -3,14 +3,14 @@ Google Compute Engine Cloud Plugin for Elasticsearch The GCE Cloud plugin allows to use GCE API for the unicast discovery mechanism. -In order to install the plugin, simply run: `bin/plugin -install elasticsearch/elasticsearch-cloud-gce/2.0.0.RC1`. +In order to install the plugin, simply run: `bin/plugin -install elasticsearch/elasticsearch-cloud-gce/2.0.0`. * For 1.0.x elasticsearch versions, look at [master branch](https://github.com/elasticsearch/elasticsearch-cloud-gce/tree/master). * For 0.90.x elasticsearch versions, look at [1.x branch](https://github.com/elasticsearch/elasticsearch-cloud-gce/tree/1.x). | GCE Cloud Plugin | elasticsearch | gce api | Release date | |----------------------------|---------------------|--------------|:------------:| -| 2.0.0-SNAPSHOT | 1.0.0.RC1 -> master | 1.17.0-rc | XXXX-XX-XX | +| 2.0.0 | 1.0.0.RC1 -> master | 1.17.0-rc | 2014-03-12 | | 2.0.0.RC1 | 1.0.0.RC1 -> master | 1.15.0-rc | 2014-01-15 | diff --git a/pom.xml b/pom.xml index e4850e88db8..6bde36e3930 100644 --- a/pom.xml +++ b/pom.xml @@ -16,7 +16,7 @@ governing permissions and limitations under the License. --> 4.0.0 org.elasticsearch elasticsearch-cloud-gce - 2.0.0-SNAPSHOT + 2.0.0 Elasticsearch Google Compute Engine cloud plugin The Google Compute Engine (GCE) Cloud plugin allows to use GCE API for the unicast discovery mechanism. From cbca1c48382bcd67c042ef383899d9d361c7171d Mon Sep 17 00:00:00 2001 From: David Pilato Date: Wed, 12 Mar 2014 12:19:17 +0100 Subject: [PATCH 38/96] prepare for next development iteration --- README.md | 1 + pom.xml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index bf52a02cf7b..d686617a9bc 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,7 @@ In order to install the plugin, simply run: `bin/plugin -install elasticsearch/e | GCE Cloud Plugin | elasticsearch | gce api | Release date | |----------------------------|---------------------|--------------|:------------:| +| 2.1.0-SNAPSHOT | 1.0.0.RC1 -> master | 1.17.0-rc | XXXX-XX-XX | | 2.0.0 | 1.0.0.RC1 -> master | 1.17.0-rc | 2014-03-12 | | 2.0.0.RC1 | 1.0.0.RC1 -> master | 1.15.0-rc | 2014-01-15 | diff --git a/pom.xml b/pom.xml index 6bde36e3930..684013d9dcd 100644 --- a/pom.xml +++ b/pom.xml @@ -16,7 +16,7 @@ governing permissions and limitations under the License. --> 4.0.0 org.elasticsearch elasticsearch-cloud-gce - 2.0.0 + 2.1.0-SNAPSHOT Elasticsearch Google Compute Engine cloud plugin The Google Compute Engine (GCE) Cloud plugin allows to use GCE API for the unicast discovery mechanism. From d71fd1f02dfd826d36d1272d93c10a57aa0dd485 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Wed, 19 Mar 2014 22:16:33 +0100 Subject: [PATCH 39/96] Disable java and maven version checking And fix typo in email html --- dev-tools/build_release.py | 27 ++++++--------------------- 1 file changed, 6 insertions(+), 21 deletions(-) diff --git a/dev-tools/build_release.py b/dev-tools/build_release.py index 9166b09e7e3..74acd8c5f4e 100755 --- a/dev-tools/build_release.py +++ b/dev-tools/build_release.py @@ -43,7 +43,7 @@ from os.path import dirname, abspath The script takes over almost all steps necessary for a release from a high level point of view it does the following things: - - run prerequisite checks ie. check for Java 1.6 being present or S3 credentials available as env variables + - run prerequisite checks ie. check for S3 credentials available as env variables - detect the version to release from the specified branch (--branch) or the current branch - creates a release branch & updates pom.xml and README.md to point to a release version rather than a snapshot - builds the artifacts @@ -109,18 +109,6 @@ def java_exe(): path = JAVA_HOME return 'export JAVA_HOME="%s" PATH="%s/bin:$PATH" JAVACMD="%s/bin/java"' % (path, path, path) -def verify_java_version(version): - s = os.popen('%s; java -version 2>&1' % java_exe()).read() - if s.find(' version "%s.' % version) == -1: - raise RuntimeError('got wrong version for java %s:\n%s' % (version, s)) - -# Verifies the java version. We guarantee that we run with Java 1.6 -# If 1.6 is not available fail the build! -def verify_mvn_java_version(version, mvn): - s = os.popen('%s; %s --version 2>&1' % (java_exe(), mvn)).read() - if s.find('Java version: %s' % version) == -1: - raise RuntimeError('got wrong java version for %s %s:\n%s' % (mvn, version, s)) - # Returns the hash of the current git HEAD revision def get_head_hash(): return os.popen(' git rev-parse --verify HEAD 2>&1').read().strip() @@ -133,9 +121,6 @@ def get_tag_hash(tag): def get_current_branch(): return os.popen('git rev-parse --abbrev-ref HEAD 2>&1').read().strip() -verify_java_version('1.6') # we require to build with 1.6 -verify_mvn_java_version('1.6', MVN) - # Utility that returns the name of the release branch for a given version def release_branch(version): return 'release_branch_%s' % version @@ -218,7 +203,7 @@ def add_version_snapshot(readme_file, release, snapshot): # If we find pattern, we copy the line and replace its content if line.find(pattern) >= 0: return line.replace(pattern, replacement).replace('%s' % (datetime.datetime.now().strftime("%Y-%m-%d")), - 'XXXX-XX-XX')+line + 'XXXX-XX-XX')+line else: return line process_file(readme_file, callback) @@ -356,15 +341,15 @@ def format_issues_html(issues, title='Fix'): if len(issues) > 0: response += '

%s

\n
    \n' % title for issue in issues: - response += '[%s] - %s\n' % (issue.html_url, issue.number, issue.title) + response += '
  • [%s] - %s\n' % (issue.html_url, issue.number, issue.title) response += '
\n' return response def get_github_repository(reponame, - login=env.get('GITHUB_LOGIN', None), - password=env.get('GITHUB_PASSWORD', None), - key=env.get('GITHUB_KEY', None)): + login=env.get('GITHUB_LOGIN', None), + password=env.get('GITHUB_PASSWORD', None), + key=env.get('GITHUB_KEY', None)): if login: g = github3.login(login, password) elif key: From c5a03cb448ec923b39f98551a2923f4ad8d8769e Mon Sep 17 00:00:00 2001 From: David Pilato Date: Wed, 26 Mar 2014 19:56:41 +0100 Subject: [PATCH 40/96] Create branches according to elasticsearch versions We create branches: * es-0.90 for elasticsearch 0.90 * es-1.0 for elasticsearch 1.0 * es-1.1 for elasticsearch 1.1 * master for elasticsearch master We also check that before releasing we don't have a dependency to an elasticsearch SNAPSHOT version. Add links to each version in documentation (cherry picked from commit a51926c) --- README.md | 12 ++++++++++-- dev-tools/build_release.py | 29 +++++++++++++++++++++++++++++ pom.xml | 2 +- 3 files changed, 40 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index d686617a9bc..f7d1d6eb6a6 100644 --- a/README.md +++ b/README.md @@ -5,8 +5,10 @@ The GCE Cloud plugin allows to use GCE API for the unicast discovery mechanism. In order to install the plugin, simply run: `bin/plugin -install elasticsearch/elasticsearch-cloud-gce/2.0.0`. -* For 1.0.x elasticsearch versions, look at [master branch](https://github.com/elasticsearch/elasticsearch-cloud-gce/tree/master). -* For 0.90.x elasticsearch versions, look at [1.x branch](https://github.com/elasticsearch/elasticsearch-cloud-gce/tree/1.x). +* For master elasticsearch versions, look at [master branch](https://github.com/elasticsearch/elasticsearch-cloud-gce/tree/master). +* For 1.1.x elasticsearch versions, look at [es-1.1 branch](https://github.com/elasticsearch/elasticsearch-cloud-gce/tree/es-1.1). +* For 1.0.x elasticsearch versions, look at [es-1.0 branch](https://github.com/elasticsearch/elasticsearch-cloud-gce/tree/es-1.0). +* For 0.90.x elasticsearch versions, look at [es-0.90 branch](https://github.com/elasticsearch/elasticsearch-cloud-gce/tree/es-0.90). | GCE Cloud Plugin | elasticsearch | gce api | Release date | |----------------------------|---------------------|--------------|:------------:| @@ -14,6 +16,12 @@ In order to install the plugin, simply run: `bin/plugin -install elasticsearch/e | 2.0.0 | 1.0.0.RC1 -> master | 1.17.0-rc | 2014-03-12 | | 2.0.0.RC1 | 1.0.0.RC1 -> master | 1.15.0-rc | 2014-01-15 | +Please read documentation relative to the version you are using: + +* [2.0.1-SNAPSHOT](https://github.com/elasticsearch/elasticsearch-cloud-azure/blob/master/README.md) +* [2.0.0](https://github.com/elasticsearch/elasticsearch-cloud-azure/blob/v2.0.0/README.md) +* [2.0.0.RC1](https://github.com/elasticsearch/elasticsearch-cloud-azure/blob/v2.0.0.RC1/README.md) + Google Compute Engine Virtual Machine Discovery =============================== diff --git a/dev-tools/build_release.py b/dev-tools/build_release.py index 74acd8c5f4e..db8345440c7 100755 --- a/dev-tools/build_release.py +++ b/dev-tools/build_release.py @@ -208,6 +208,29 @@ def add_version_snapshot(readme_file, release, snapshot): return line process_file(readme_file, callback) +# Moves the README.md file from a snapshot to a release (documentation link) +def remove_documentation_snapshot(readme_file, repo_url, release, branch): + pattern = '* [%s-SNAPSHOT](%sblob/%s/README.md)' % (release, repo_url, branch) + replacement = '* [%s](%sblob/v%s/README.md)' % (release, repo_url, release) + def callback(line): + # If we find pattern, we replace its content + if line.find(pattern) >= 0: + return line.replace(pattern, replacement) + else: + return line + process_file(readme_file, callback) + +# Add in README.markdown file the documentation for the next version +def add_documentation_snapshot(readme_file, repo_url, release, snapshot, branch): + pattern = '* [%s](%sblob/v%s/README.md)' % (release, repo_url, release) + replacement = '* [%s-SNAPSHOT](%sblob/%s/README.md)' % (snapshot, repo_url, branch) + def callback(line): + # If we find pattern, we copy the line and replace its content + if line.find(pattern) >= 0: + return line.replace(pattern, replacement)+line + else: + return line + process_file(readme_file, callback) # Set release date in README.md file def set_date(readme_file): @@ -603,8 +626,12 @@ if __name__ == '__main__': artifact_name = find_from_pom('name') artifact_description = find_from_pom('description') project_url = find_from_pom('url') + elasticsearch_version = find_from_pom('elasticsearch.version') print(' Artifact Id: [%s]' % artifact_id) print(' Release version: [%s]' % release_version) + print(' Elasticsearch: [%s]' % elasticsearch_version) + if elasticsearch_version.find('-SNAPSHOT') != -1: + raise RuntimeError('Can not release with a SNAPSHOT elasticsearch dependency: %s' % elasticsearch_version) # extract snapshot default_snapshot_version = guess_snapshot(release_version) @@ -626,6 +653,7 @@ if __name__ == '__main__': try: pending_files = [POM_FILE, README_FILE] remove_maven_snapshot(POM_FILE, release_version) + remove_documentation_snapshot(README_FILE, project_url, release_version, src_branch) remove_version_snapshot(README_FILE, release_version) set_date(README_FILE) set_install_instructions(README_FILE, artifact_id, release_version) @@ -657,6 +685,7 @@ if __name__ == '__main__': add_maven_snapshot(POM_FILE, release_version, snapshot_version) add_version_snapshot(README_FILE, release_version, snapshot_version) + add_documentation_snapshot(README_FILE, project_url, release_version, snapshot_version, src_branch) add_pending_files(*pending_files) commit_snapshot() diff --git a/pom.xml b/pom.xml index 684013d9dcd..ff24fc004cf 100644 --- a/pom.xml +++ b/pom.xml @@ -16,7 +16,7 @@ governing permissions and limitations under the License. --> 4.0.0 org.elasticsearch elasticsearch-cloud-gce - 2.1.0-SNAPSHOT + 2.0.1-SNAPSHOT Elasticsearch Google Compute Engine cloud plugin The Google Compute Engine (GCE) Cloud plugin allows to use GCE API for the unicast discovery mechanism. From 4f04ded8522075956cd9b5f4bf3c819f3b467b0c Mon Sep 17 00:00:00 2001 From: David Pilato Date: Wed, 26 Mar 2014 20:09:14 +0100 Subject: [PATCH 41/96] Run tests with only one JVM (cherry picked from commit d003f39) --- pom.xml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index ff24fc004cf..5ca06d2c065 100644 --- a/pom.xml +++ b/pom.xml @@ -52,7 +52,6 @@ governing permissions and limitations under the License. --> v1beta16-rev1-1.17.0-rc 1.17.0-rc 4.6.1 - 1 true onerror @@ -201,7 +200,7 @@ governing permissions and limitations under the License. --> - ${tests.jvms} + 1 From 3643233670f9aca9a1315cefeeadc494885d9e28 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Wed, 26 Mar 2014 20:16:11 +0100 Subject: [PATCH 42/96] Update to elasticsearch master --- README.md | 8 ++------ pom.xml | 6 +++--- .../org/elasticsearch/discovery/gce/GceDiscovery.java | 6 ++++-- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index f7d1d6eb6a6..d8c4048bc2c 100644 --- a/README.md +++ b/README.md @@ -12,15 +12,11 @@ In order to install the plugin, simply run: `bin/plugin -install elasticsearch/e | GCE Cloud Plugin | elasticsearch | gce api | Release date | |----------------------------|---------------------|--------------|:------------:| -| 2.1.0-SNAPSHOT | 1.0.0.RC1 -> master | 1.17.0-rc | XXXX-XX-XX | -| 2.0.0 | 1.0.0.RC1 -> master | 1.17.0-rc | 2014-03-12 | -| 2.0.0.RC1 | 1.0.0.RC1 -> master | 1.15.0-rc | 2014-01-15 | +| 3.0.0-SNAPSHOT | master | 1.17.0-rc | XXXX-XX-XX | Please read documentation relative to the version you are using: -* [2.0.1-SNAPSHOT](https://github.com/elasticsearch/elasticsearch-cloud-azure/blob/master/README.md) -* [2.0.0](https://github.com/elasticsearch/elasticsearch-cloud-azure/blob/v2.0.0/README.md) -* [2.0.0.RC1](https://github.com/elasticsearch/elasticsearch-cloud-azure/blob/v2.0.0.RC1/README.md) +* [3.0.0-SNAPSHOT](https://github.com/elasticsearch/elasticsearch-cloud-azure/blob/master/README.md) Google Compute Engine Virtual Machine Discovery diff --git a/pom.xml b/pom.xml index 5ca06d2c065..80ec5cd3bfe 100644 --- a/pom.xml +++ b/pom.xml @@ -16,7 +16,7 @@ governing permissions and limitations under the License. --> 4.0.0 org.elasticsearch elasticsearch-cloud-gce - 2.0.1-SNAPSHOT + 3.0.0-SNAPSHOT Elasticsearch Google Compute Engine cloud plugin The Google Compute Engine (GCE) Cloud plugin allows to use GCE API for the unicast discovery mechanism. @@ -48,10 +48,10 @@ governing permissions and limitations under the License. --> - 1.0.1 + 2.0.0-SNAPSHOT v1beta16-rev1-1.17.0-rc 1.17.0-rc - 4.6.1 + 4.7.0 true onerror diff --git a/src/main/java/org/elasticsearch/discovery/gce/GceDiscovery.java b/src/main/java/org/elasticsearch/discovery/gce/GceDiscovery.java index cf82ac1f78f..3c4ab62896d 100755 --- a/src/main/java/org/elasticsearch/discovery/gce/GceDiscovery.java +++ b/src/main/java/org/elasticsearch/discovery/gce/GceDiscovery.java @@ -28,6 +28,7 @@ import org.elasticsearch.common.collect.ImmutableList; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.network.NetworkService; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.discovery.DiscoverySettings; import org.elasticsearch.discovery.zen.ZenDiscovery; import org.elasticsearch.discovery.zen.ping.ZenPing; import org.elasticsearch.discovery.zen.ping.ZenPingService; @@ -45,8 +46,9 @@ public class GceDiscovery extends ZenDiscovery { public GceDiscovery(Settings settings, ClusterName clusterName, ThreadPool threadPool, TransportService transportService, ClusterService clusterService, NodeSettingsService nodeSettingsService, ZenPingService pingService, DiscoveryNodeService discoveryNodeService, GceComputeService gceComputeService, - NetworkService networkService) { - super(settings, clusterName, threadPool, transportService, clusterService, nodeSettingsService, discoveryNodeService, pingService, Version.CURRENT); + NetworkService networkService, DiscoverySettings discoverySettings) { + super(settings, clusterName, threadPool, transportService, clusterService, nodeSettingsService, + discoveryNodeService, pingService, Version.CURRENT, discoverySettings); if (settings.getAsBoolean("cloud.enabled", true)) { ImmutableList zenPings = pingService.zenPings(); UnicastZenPing unicastZenPing = null; From a181b711f2ddb825204b204fdd193eb235ddd652 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Wed, 26 Mar 2014 20:16:11 +0100 Subject: [PATCH 43/96] Fix typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d8c4048bc2c..dd778f625d1 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ In order to install the plugin, simply run: `bin/plugin -install elasticsearch/e Please read documentation relative to the version you are using: -* [3.0.0-SNAPSHOT](https://github.com/elasticsearch/elasticsearch-cloud-azure/blob/master/README.md) +* [3.0.0-SNAPSHOT](https://github.com/elasticsearch/elasticsearch-cloud-gce/blob/master/README.md) Google Compute Engine Virtual Machine Discovery From 290fb027c33ac2dafad6f2382615c65d21dac5ea Mon Sep 17 00:00:00 2001 From: David Pilato Date: Mon, 14 Apr 2014 23:36:24 +0200 Subject: [PATCH 44/96] Upgrade to GCE v1-rev15-1.18.0-rc API Closes #19. --- README.md | 2 +- pom.xml | 4 ++-- .../elasticsearch/discovery/gce/GceUnicastHostsProvider.java | 5 ++--- .../discovery/gce/GceComputeServiceAbstractMock.java | 1 - 4 files changed, 5 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index dd778f625d1..9cd8af83396 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ In order to install the plugin, simply run: `bin/plugin -install elasticsearch/e | GCE Cloud Plugin | elasticsearch | gce api | Release date | |----------------------------|---------------------|--------------|:------------:| -| 3.0.0-SNAPSHOT | master | 1.17.0-rc | XXXX-XX-XX | +| 3.0.0-SNAPSHOT | master | 1.18.0-rc | XXXX-XX-XX | Please read documentation relative to the version you are using: diff --git a/pom.xml b/pom.xml index 80ec5cd3bfe..722ef498c2e 100644 --- a/pom.xml +++ b/pom.xml @@ -49,8 +49,8 @@ governing permissions and limitations under the License. --> 2.0.0-SNAPSHOT - v1beta16-rev1-1.17.0-rc - 1.17.0-rc + v1-rev15-1.18.0-rc + 1.18.0-rc 4.7.0 true onerror diff --git a/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java b/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java index a0238b00b4f..2a6247e860a 100644 --- a/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java +++ b/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java @@ -130,7 +130,6 @@ public class GceUnicastHostsProvider extends AbstractComponent implements Unicas for (Instance instance : instances) { String name = instance.getName(); String type = instance.getMachineType(); - String image = instance.getImage(); String status = instance.getStatus(); logger.trace("gce instance {} with status {} found.", name, status); @@ -227,8 +226,8 @@ public class GceUnicastHostsProvider extends AbstractComponent implements Unicas // If user has set `es_port` metadata, we don't need to ping all ports // we only limit to 1 addresses, makes no sense to ping 100 ports for (int i = 0; i < addresses.length; i++) { - logger.trace("adding {}, type {}, image {}, address {}, transport_address {}, status {}", name, type, - image, ip_private, addresses[i], status); + logger.trace("adding {}, type {}, image {}, address {}, transport_address {}, status {}", name, type + , ip_private, addresses[i], status); cachedDiscoNodes.add(new DiscoveryNode("#cloud-" + name + "-" + i, addresses[i], Version.CURRENT)); } } diff --git a/src/test/java/org/elasticsearch/discovery/gce/GceComputeServiceAbstractMock.java b/src/test/java/org/elasticsearch/discovery/gce/GceComputeServiceAbstractMock.java index 4a13466b0c6..1cbe9bcc55a 100644 --- a/src/test/java/org/elasticsearch/discovery/gce/GceComputeServiceAbstractMock.java +++ b/src/test/java/org/elasticsearch/discovery/gce/GceComputeServiceAbstractMock.java @@ -60,7 +60,6 @@ public abstract class GceComputeServiceAbstractMock extends AbstractLifecycleCom Instance instance = new Instance(); instance.setName("Mock Node " + tags); instance.setMachineType("Mock Type machine"); - instance.setImage("mock-image-type"); instance.setStatus("STARTED"); Tags instanceTags = new Tags(); instanceTags.setItems(tags); From 89c29c44044ed41e53fa24c64f8f2288347f50a1 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Mon, 14 Apr 2014 23:38:33 +0200 Subject: [PATCH 45/96] Fix log Related to #19. --- .../elasticsearch/discovery/gce/GceUnicastHostsProvider.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java b/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java index 2a6247e860a..c6981de9bef 100644 --- a/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java +++ b/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java @@ -226,8 +226,8 @@ public class GceUnicastHostsProvider extends AbstractComponent implements Unicas // If user has set `es_port` metadata, we don't need to ping all ports // we only limit to 1 addresses, makes no sense to ping 100 ports for (int i = 0; i < addresses.length; i++) { - logger.trace("adding {}, type {}, image {}, address {}, transport_address {}, status {}", name, type - , ip_private, addresses[i], status); + logger.trace("adding {}, type {}, address {}, transport_address {}, status {}", name, type, + ip_private, addresses[i], status); cachedDiscoNodes.add(new DiscoveryNode("#cloud-" + name + "-" + i, addresses[i], Version.CURRENT)); } } From c0d047dccb153bd6884ef0d94b21821855ecf1d0 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Wed, 16 Apr 2014 08:00:52 +0200 Subject: [PATCH 46/96] Fix tests when running mvn tests (cherry picked from commit 1edd72e) --- pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/pom.xml b/pom.xml index 722ef498c2e..e75719857e5 100644 --- a/pom.xml +++ b/pom.xml @@ -57,6 +57,7 @@ governing permissions and limitations under the License. --> 9300 INFO + ${project.build.testOutputDirectory}/elasticsearch.yml From 314b146fd54236e716d6e750d2527c899862e3d9 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Tue, 29 Apr 2014 21:59:10 +0200 Subject: [PATCH 47/96] Create branch es-1.2 --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 9cd8af83396..ee26d4fb7a7 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,7 @@ The GCE Cloud plugin allows to use GCE API for the unicast discovery mechanism. In order to install the plugin, simply run: `bin/plugin -install elasticsearch/elasticsearch-cloud-gce/2.0.0`. * For master elasticsearch versions, look at [master branch](https://github.com/elasticsearch/elasticsearch-cloud-gce/tree/master). +* For 1.2.x elasticsearch versions, look at [es-1.2 branch](https://github.com/elasticsearch/elasticsearch-cloud-gce/tree/es-1.2). * For 1.1.x elasticsearch versions, look at [es-1.1 branch](https://github.com/elasticsearch/elasticsearch-cloud-gce/tree/es-1.1). * For 1.0.x elasticsearch versions, look at [es-1.0 branch](https://github.com/elasticsearch/elasticsearch-cloud-gce/tree/es-1.0). * For 0.90.x elasticsearch versions, look at [es-0.90 branch](https://github.com/elasticsearch/elasticsearch-cloud-gce/tree/es-0.90). From 7d7b2241e5d5d5522bffa84f4eb1a2a183456632 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Tue, 29 Apr 2014 22:04:02 +0200 Subject: [PATCH 48/96] Update to Lucene 4.8.0/ elasticsearch 2.0.0 Latest changes break tests Closes #21. (cherry picked from commit 04c77e8) --- pom.xml | 2 +- .../discovery/gce/AbstractGceComputeServiceTest.java | 2 +- src/test/java/org/elasticsearch/gce/itest/GceSimpleITest.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index e75719857e5..1f9eeca053d 100644 --- a/pom.xml +++ b/pom.xml @@ -51,7 +51,7 @@ governing permissions and limitations under the License. --> 2.0.0-SNAPSHOT v1-rev15-1.18.0-rc 1.18.0-rc - 4.7.0 + 4.8.0 true onerror diff --git a/src/test/java/org/elasticsearch/discovery/gce/AbstractGceComputeServiceTest.java b/src/test/java/org/elasticsearch/discovery/gce/AbstractGceComputeServiceTest.java index a7c4220781b..57980ffebf9 100644 --- a/src/test/java/org/elasticsearch/discovery/gce/AbstractGceComputeServiceTest.java +++ b/src/test/java/org/elasticsearch/discovery/gce/AbstractGceComputeServiceTest.java @@ -32,7 +32,7 @@ import static org.hamcrest.Matchers.is; @ElasticsearchIntegrationTest.ClusterScope( scope = ElasticsearchIntegrationTest.Scope.SUITE, - numNodes = 2, + numDataNodes = 2, transportClientRatio = 0.0) public abstract class AbstractGceComputeServiceTest extends ElasticsearchIntegrationTest { /** diff --git a/src/test/java/org/elasticsearch/gce/itest/GceSimpleITest.java b/src/test/java/org/elasticsearch/gce/itest/GceSimpleITest.java index 5e2accbe827..48d0c1a762e 100644 --- a/src/test/java/org/elasticsearch/gce/itest/GceSimpleITest.java +++ b/src/test/java/org/elasticsearch/gce/itest/GceSimpleITest.java @@ -37,7 +37,7 @@ import org.junit.Test; @AbstractGceTest.GceTest @ElasticsearchIntegrationTest.ClusterScope( scope = ElasticsearchIntegrationTest.Scope.TEST, - numNodes = 1, + numDataNodes = 1, transportClientRatio = 0.0) public class GceSimpleITest extends AbstractGceTest { From b1e27bbb32f015bf82b04ba71f13967c6a035fba Mon Sep 17 00:00:00 2001 From: David Pilato Date: Sun, 15 Jun 2014 10:58:54 +0200 Subject: [PATCH 49/96] NPE when no tags on instance When no tags exists on other running instances and if we try to filter by tag, we get the following error: ``` [2014-05-19 16:17:37,377][DEBUG][discovery.gce ] [Theresa Cassidy] start building nodes list using GCE API [2014-05-19 16:17:37,378][INFO ][cloud.gce ] [Theresa Cassidy] starting GCE discovery service [2014-05-19 16:17:37,592][TRACE][discovery.gce ] [Theresa Cassidy] gce instance hadoop1 with status RUNNING found. [2014-05-19 16:17:37,597][TRACE][discovery.gce ] [Theresa Cassidy] start filtering instance hadoop1 with tags [elasticsearch, dev]. [2014-05-19 16:17:37,597][TRACE][discovery.gce ] [Theresa Cassidy] comparing instance tags null with tags filter [elasticsearch, dev]. [2014-05-19 16:17:37,597][WARN ][discovery.gce ] [Theresa Cassidy] Exception caught during discovery java.lang.NullPointerException : null [2014-05-19 16:17:37,597][TRACE][discovery.gce ] [Theresa Cassidy] Exception caught during discovery java.lang.NullPointerException at org.elasticsearch.discovery.gce.GceUnicastHostsProvider.buildDynamicNodes(GceUnicastHostsProvider.java:157) at org.elasticsearch.discovery.zen.ping.unicast.UnicastZenPing.sendPings(UnicastZenPing.java:245) at org.elasticsearch.discovery.zen.ping.unicast.UnicastZenPing$2.run(UnicastZenPing.java:176) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:744) [2014-05-19 16:17:37,598][DEBUG][discovery.gce ] [Theresa Cassidy] 0 node(s) added [2014-05-19 16:17:37,598][DEBUG][discovery.gce ] [Theresa Cassidy] using dynamic discovery nodes [] ``` Closes #22. --- .../discovery/gce/GceUnicastHostsProvider.java | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java b/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java index c6981de9bef..d2508d4ffd4 100644 --- a/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java +++ b/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java @@ -145,7 +145,8 @@ public class GceUnicastHostsProvider extends AbstractComponent implements Unicas boolean filterByTag = false; if (tags.length > 0) { logger.trace("start filtering instance {} with tags {}.", name, tags); - if (instance.getTags() == null || instance.getTags().isEmpty()) { + if (instance.getTags() == null || instance.getTags().isEmpty() + || instance.getTags().getItems() == null || instance.getTags().getItems().isEmpty()) { // If this instance have no tag, we filter it logger.trace("no tags for this instance but we asked for tags. {} won't be part of the cluster.", name); filterByTag = true; @@ -169,7 +170,7 @@ public class GceUnicastHostsProvider extends AbstractComponent implements Unicas } if (filterByTag) { logger.trace("filtering out instance {} based tags {}, not part of {}", name, tags, - instance.getTags() == null ? "" : instance.getTags().getItems()); + instance.getTags() == null || instance.getTags().getItems() == null ? "" : ""); continue; } else { logger.trace("instance {} with tags {} is added to discovery", name, tags); @@ -225,11 +226,9 @@ public class GceUnicastHostsProvider extends AbstractComponent implements Unicas // If user has set `es_port` metadata, we don't need to ping all ports // we only limit to 1 addresses, makes no sense to ping 100 ports - for (int i = 0; i < addresses.length; i++) { - logger.trace("adding {}, type {}, address {}, transport_address {}, status {}", name, type, - ip_private, addresses[i], status); - cachedDiscoNodes.add(new DiscoveryNode("#cloud-" + name + "-" + i, addresses[i], Version.CURRENT)); - } + logger.trace("adding {}, type {}, address {}, transport_address {}, status {}", name, type, + ip_private, addresses[0], status); + cachedDiscoNodes.add(new DiscoveryNode("#cloud-" + name + "-" + 0, addresses[0], Version.CURRENT)); } } catch (Exception e) { logger.warn("failed to add {}, address {}", e, name, ip_private); From 37474b43d6fbc49ed91907194807254ef88c546f Mon Sep 17 00:00:00 2001 From: David Pilato Date: Sun, 15 Jun 2014 10:57:43 +0200 Subject: [PATCH 50/96] Update instructions for GCE Some tools have changed on GCE platform. We need to update the documentation. Closes #23. --- README.md | 113 +++++++++++++++++++++--------------------------------- 1 file changed, 43 insertions(+), 70 deletions(-) diff --git a/README.md b/README.md index ee26d4fb7a7..55e83822ba6 100644 --- a/README.md +++ b/README.md @@ -52,14 +52,19 @@ How to start (long story) Before starting, you should have: * Your project ID. Let's say here `es-cloud`. Get it from [Google APIS Console](https://code.google.com/apis/console/). -* [GCUtil](https://developers.google.com/compute/docs/gcutil/#install) +* [Google Cloud SDK](https://developers.google.com/cloud/sdk/) +If you did not set it yet, you can define your default project you will work on: + +```sh +gcloud config set project es-cloud +``` ### Creating your first instance ```sh -gcutil --project=es-cloud addinstance myesnode1 \ +gcutil addinstance myesnode1 \ --service_account_scope=compute-rw,storage-full \ --persistent_boot_disk ``` @@ -73,7 +78,7 @@ Then, choose your zone. Let's say here that we choose `europe-west1-a`. Choose your compute instance size. Let's say `f1-micro`. -Choose your OS. Let's say `projects/debian-cloud/global/images/debian-7-wheezy-v20130617`. +Choose your OS. Let's say `projects/debian-cloud/global/images/debian-7-wheezy-v20140606`. You may be asked to create a ssh key. Follow instructions to create one. @@ -89,9 +94,13 @@ Table of resources: +-----------+--------------+-------+---------+--------------+----------------+----------------+----------------+---------+----------------+ ``` -You can now connect to your machine using the external IP address in order to install Elasticsearch: +You can now connect to your instance: ``` +# Connect using google cloud SDK +gcloud compute ssh myesnode1 --zone europe-west1-a + +# Or using SSH with external IP address ssh -i ~/.ssh/google_compute_engine 192.158.29.199 ``` @@ -100,38 +109,14 @@ Once connected, install Elasticsearch: ```sh sudo apt-get update -# Install curl if needed -sudo apt-get install curl - # Download Elasticsearch -wget https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-1.0.0.RC1.deb +wget https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-1.2.1.deb # Prepare Java installation sudo apt-get install java7-runtime-headless # Prepare Elasticsearch installation -sudo dpkg -i elasticsearch-1.0.0.RC1.deb - -# Check that elasticsearch is running: -curl http://localhost:9200/ -``` - -This command should give you a JSON result: - -```javascript -{ - "ok" : true, - "status" : 200, - "name" : "Shriker", - "version" : { - "number" : "1.0.0.RC1", - "build_hash" : "0a5781f44876e8d1c30b6360628d59cb2a7a2bbb", - "build_timestamp" : "2014-01-10T10:18:37Z", - "build_snapshot" : false, - "lucene_version" : "4.6" - }, - "tagline" : "You Know, for Search" -} +sudo dpkg -i elasticsearch-1.2.1.deb ``` ### Install elasticsearch cloud gce plugin @@ -140,7 +125,7 @@ Install the plugin: ```sh # Use Plugin Manager to install it -sudo /usr/share/elasticsearch/bin/plugin --install elasticsearch/elasticsearch-cloud-gce/2.0.0.RC1 +sudo /usr/share/elasticsearch/bin/plugin --install elasticsearch/elasticsearch-cloud-gce/2.2.0 # Configure it: sudo vi /etc/elasticsearch/elasticsearch.yml @@ -149,19 +134,19 @@ sudo vi /etc/elasticsearch/elasticsearch.yml And add the following lines: ```yaml - cloud: - gce: - project_id: es-cloud - zone: europe-west1-a - discovery: - type: gce +cloud: + gce: + project_id: es-cloud + zone: europe-west1-a +discovery: + type: gce ``` -Restart elasticsearch: +Start elasticsearch: ```sh -sudo /etc/init.d/elasticsearch restart +sudo /etc/init.d/elasticsearch start ``` If anything goes wrong, you should check logs: @@ -188,17 +173,13 @@ First create an image of your running instance and upload it to Google Cloud Sto ```sh # Create an image of yur current instance -sudo python /usr/share/imagebundle/image_bundle.py \ - -r / -o /tmp/ --log_file=/tmp/abc.log +sudo /usr/bin/gcimagebundle -d /dev/sda -o /tmp/ # An image has been created in `/tmp` directory: ls /tmp e4686d7f5bf904a924ae0cfeb58d0827c6d5b966.image.tar.gz # Upload your image to Google Cloud Storage: -# Launch this command and follow instructions to give your instance an access to your storage -gsutil config - # Create a bucket to hold your image, let's say `esimage`: gsutil mb gs://esimage @@ -206,17 +187,7 @@ gsutil mb gs://esimage gsutil cp /tmp/e4686d7f5bf904a924ae0cfeb58d0827c6d5b966.image.tar.gz gs://esimage # Then add your image to images collection: -gcutil listkernels --project es-cloud -+----------------------------------------------+--------------------------------------------------+-------------+ -| name | description | deprecation | -+----------------------------------------------+--------------------------------------------------+-------------+ -| projects/google/global/kernels/gce-20120621 | 2.6.39-gcg built 2012-03-29 01:07:00 | DEPRECATED | -| projects/google/global/kernels/gce-v20130603 | SCSI-enabled 3.3.8-gcg built 2013-05-29 01:04:00 | | -+----------------------------------------------+--------------------------------------------------+-------------+ -# Note the kernel you prefer to use and add your image to your catalog: -gcutil --project=es-cloud addimage elasticsearch-1-0-0-RC1 \ - gs://esimage/e4686d7f5bf904a924ae0cfeb58d0827c6d5b966.image.tar.gz \ - --preferred_kernel=projects/google/global/kernels/gce-v20130603 +gcutil addimage elasticsearch-1-2-1 gs://esimage/e4686d7f5bf904a924ae0cfeb58d0827c6d5b966.image.tar.gz # If the previous command did not work for you, logout from your instance # and launch the same command from your local machine. @@ -228,7 +199,10 @@ As you have now an image, you can create as many instances as you need: ```sh # Just change node name (here myesnode2) -gcutil --project=es-cloud addinstance --image=elasticsearch-1-0-0-RC1 \ +gcutil addinstance --image=elasticsearch-1-2-1 myesnode2 + +# If you want to provide all details directly, you can use: +gcutil addinstance --image=elasticsearch-1-2-1 \ --kernel=projects/google/global/kernels/gce-v20130603 myesnode2 \ --zone europe-west1-a --machine_type f1-micro --service_account_scope=compute-rw \ --persistent_boot_disk @@ -240,11 +214,11 @@ You can use [Google Cloud Console](https://cloud.google.com/console) or CLI to m ```sh # Stopping and removing instances -gcutil --project=es-cloud deleteinstance myesnode1 myesnode2 \ +gcutil deleteinstance myesnode1 myesnode2 \ --zone=europe-west1-a # Consider removing disk as well if you don't need them anymore -gcutil --project=es-cloud deletedisk boot-myesnode1 boot-myesnode2 \ +gcutil deletedisk boot-myesnode1 boot-myesnode2 \ --zone=europe-west1-a ``` @@ -273,14 +247,14 @@ gcutil --project=es-cloud addinstance myesnode1 \ Then, define it in `elasticsearch.yml`: ```yaml - cloud: +cloud: + gce: + project_id: es-cloud + zone: europe-west1-a +discovery: + type: gce gce: - project_id: es-cloud - zone: europe-west1-a - discovery: - type: gce - gce: - tags: elasticsearch, dev + tags: elasticsearch, dev ``` Changing default transport port @@ -295,13 +269,13 @@ Add `--metadata=es_port:9301` option: ```sh # when creating first instance -gcutil --project=es-cloud addinstance myesnode1 \ +gcutil addinstance myesnode1 \ --service_account_scope=compute-rw,storage-full \ --persistent_boot_disk \ --metadata=es_port:9301 # when creating an instance from an image -gcutil --project=es-cloud addinstance --image=elasticsearch-1-0-0-RC1 \ +gcutil addinstance --image=elasticsearch-1-0-0-RC1 \ --kernel=projects/google/global/kernels/gce-v20130603 myesnode2 \ --zone europe-west1-a --machine_type f1-micro --service_account_scope=compute-rw \ --persistent_boot_disk --metadata=es_port:9301 @@ -311,8 +285,7 @@ gcutil --project=es-cloud addinstance --image=elasticsearch-1-0-0-RC1 \ ```sh # Get metadata fingerprint -gcutil --project=es-cloud getinstance myesnode1 \ - --zone=europe-west1-a +gcutil getinstance myesnode1 --zone=europe-west1-a +------------------------+---------------------------------------------------------------------------------------------------------+ | property | value | +------------------------+---------------------------------------------------------------------------------------------------------+ @@ -321,7 +294,7 @@ gcutil --project=es-cloud getinstance myesnode1 \ +------------------------+---------------------------------------------------------------------------------------------------------+ # Use that fingerprint -gcutil --project=es-cloud setinstancemetadata myesnode1 \ +gcutil setinstancemetadata myesnode1 \ --zone=europe-west1-a \ --metadata=es_port:9301 \ --fingerprint=42WmSpB8rSM= From 81742c3c1d5c33cb13b688dec0c580367d2b4878 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Sun, 15 Jun 2014 11:17:47 +0200 Subject: [PATCH 51/96] Update to elasticsearch 1.3.0 Closes #24. (cherry picked from commit 9c62411) --- README.md | 3 ++- pom.xml | 2 +- .../discovery/gce/AbstractGceComputeServiceTest.java | 2 ++ .../org/elasticsearch/gce/itest/GceSimpleITest.java | 11 ++++++++++- 4 files changed, 15 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 55e83822ba6..77ec97e2e35 100644 --- a/README.md +++ b/README.md @@ -3,9 +3,10 @@ Google Compute Engine Cloud Plugin for Elasticsearch The GCE Cloud plugin allows to use GCE API for the unicast discovery mechanism. -In order to install the plugin, simply run: `bin/plugin -install elasticsearch/elasticsearch-cloud-gce/2.0.0`. +In order to install the plugin, simply run: `bin/plugin -install elasticsearch/elasticsearch-cloud-gce/2.1.1`. * For master elasticsearch versions, look at [master branch](https://github.com/elasticsearch/elasticsearch-cloud-gce/tree/master). +* For 1.3.x elasticsearch versions, look at [es-1.3 branch](https://github.com/elasticsearch/elasticsearch-cloud-gce/tree/es-1.3). * For 1.2.x elasticsearch versions, look at [es-1.2 branch](https://github.com/elasticsearch/elasticsearch-cloud-gce/tree/es-1.2). * For 1.1.x elasticsearch versions, look at [es-1.1 branch](https://github.com/elasticsearch/elasticsearch-cloud-gce/tree/es-1.1). * For 1.0.x elasticsearch versions, look at [es-1.0 branch](https://github.com/elasticsearch/elasticsearch-cloud-gce/tree/es-1.0). diff --git a/pom.xml b/pom.xml index 1f9eeca053d..ad91d70da13 100644 --- a/pom.xml +++ b/pom.xml @@ -51,7 +51,7 @@ governing permissions and limitations under the License. --> 2.0.0-SNAPSHOT v1-rev15-1.18.0-rc 1.18.0-rc - 4.8.0 + 4.8.1 true onerror diff --git a/src/test/java/org/elasticsearch/discovery/gce/AbstractGceComputeServiceTest.java b/src/test/java/org/elasticsearch/discovery/gce/AbstractGceComputeServiceTest.java index 57980ffebf9..a39b500a8f4 100644 --- a/src/test/java/org/elasticsearch/discovery/gce/AbstractGceComputeServiceTest.java +++ b/src/test/java/org/elasticsearch/discovery/gce/AbstractGceComputeServiceTest.java @@ -22,6 +22,7 @@ package org.elasticsearch.discovery.gce; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; import org.elasticsearch.common.settings.ImmutableSettings; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.test.ElasticsearchIntegrationTest; import org.junit.Test; @@ -55,6 +56,7 @@ public abstract class AbstractGceComputeServiceTest extends ElasticsearchIntegra return ImmutableSettings.builder() .put("transport.tcp.port", getPort(nodeOrdinal)) .put("http.enabled", false) + .put("plugins." + PluginsService.LOAD_PLUGIN_FROM_CLASSPATH, true) .build(); } diff --git a/src/test/java/org/elasticsearch/gce/itest/GceSimpleITest.java b/src/test/java/org/elasticsearch/gce/itest/GceSimpleITest.java index 48d0c1a762e..f02aecc543b 100644 --- a/src/test/java/org/elasticsearch/gce/itest/GceSimpleITest.java +++ b/src/test/java/org/elasticsearch/gce/itest/GceSimpleITest.java @@ -23,6 +23,7 @@ import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; import org.elasticsearch.cloud.gce.AbstractGceTest; import org.elasticsearch.common.settings.ImmutableSettings; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.test.ElasticsearchIntegrationTest; import org.hamcrest.Matchers; import org.junit.Test; @@ -36,11 +37,19 @@ import org.junit.Test; */ @AbstractGceTest.GceTest @ElasticsearchIntegrationTest.ClusterScope( - scope = ElasticsearchIntegrationTest.Scope.TEST, + scope = ElasticsearchIntegrationTest.Scope.SUITE, numDataNodes = 1, transportClientRatio = 0.0) public class GceSimpleITest extends AbstractGceTest { + @Override + protected Settings nodeSettings(int nodeOrdinal) { + return ImmutableSettings.builder() + .put(super.nodeSettings(nodeOrdinal)) + .put("plugins." + PluginsService.LOAD_PLUGIN_FROM_CLASSPATH, true) + .build(); + } + @Test public void one_node_should_run() { // Do nothing... Just start :-) From e35e51556ea49810dd7af64eb88e17fe448e3810 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Sun, 15 Jun 2014 23:28:15 +0200 Subject: [PATCH 52/96] Fix tests When upgrading to elasticsearch 1.2.1 test framework, some tests are not working anymore because of `ElasticsearchIntegrationTest#ensureClusterSizeConsistency()` method which check that the number of started nodes is the number of available nodes in the cluster. Disabling them temporary. Also, a new clientNode could be added (depends on seed). It adds a node more than expected. (cherry picked from commit bcc2cd5) --- pom.xml | 26 ++++++++++--------- .../gce/AbstractGceComputeServiceTest.java | 2 ++ .../gce/GceDifferentTagsOneTagTest.java | 8 ++++++ .../gce/GceDifferentTagsTwoTagsTest.java | 8 ++++++ 4 files changed, 32 insertions(+), 12 deletions(-) diff --git a/pom.xml b/pom.xml index ad91d70da13..8ac04aa62cd 100644 --- a/pom.xml +++ b/pom.xml @@ -61,6 +61,19 @@ governing permissions and limitations under the License. --> + + org.hamcrest + hamcrest-core + 1.3.RC2 + test + + + org.hamcrest + hamcrest-library + 1.3.RC2 + test + + org.apache.lucene lucene-test-framework @@ -104,6 +117,7 @@ governing permissions and limitations under the License. --> 1.2.17 test + org.elasticsearch elasticsearch @@ -111,18 +125,6 @@ governing permissions and limitations under the License. --> test-jar test - - org.hamcrest - hamcrest-core - 1.3.RC2 - test - - - org.hamcrest - hamcrest-library - 1.3.RC2 - test - diff --git a/src/test/java/org/elasticsearch/discovery/gce/AbstractGceComputeServiceTest.java b/src/test/java/org/elasticsearch/discovery/gce/AbstractGceComputeServiceTest.java index a39b500a8f4..ac4d6d7817b 100644 --- a/src/test/java/org/elasticsearch/discovery/gce/AbstractGceComputeServiceTest.java +++ b/src/test/java/org/elasticsearch/discovery/gce/AbstractGceComputeServiceTest.java @@ -34,6 +34,7 @@ import static org.hamcrest.Matchers.is; @ElasticsearchIntegrationTest.ClusterScope( scope = ElasticsearchIntegrationTest.Scope.SUITE, numDataNodes = 2, + numClientNodes = 0, transportClientRatio = 0.0) public abstract class AbstractGceComputeServiceTest extends ElasticsearchIntegrationTest { /** @@ -57,6 +58,7 @@ public abstract class AbstractGceComputeServiceTest extends ElasticsearchIntegra .put("transport.tcp.port", getPort(nodeOrdinal)) .put("http.enabled", false) .put("plugins." + PluginsService.LOAD_PLUGIN_FROM_CLASSPATH, true) + .put(super.nodeSettings(nodeOrdinal)) .build(); } diff --git a/src/test/java/org/elasticsearch/discovery/gce/GceDifferentTagsOneTagTest.java b/src/test/java/org/elasticsearch/discovery/gce/GceDifferentTagsOneTagTest.java index 8ccb98f6fd7..b53a9e30ebd 100644 --- a/src/test/java/org/elasticsearch/discovery/gce/GceDifferentTagsOneTagTest.java +++ b/src/test/java/org/elasticsearch/discovery/gce/GceDifferentTagsOneTagTest.java @@ -21,7 +21,15 @@ package org.elasticsearch.discovery.gce; import org.elasticsearch.common.settings.ImmutableSettings; import org.elasticsearch.common.settings.Settings; +import org.junit.Ignore; +/** + * We need to ignore this test from elasticsearch version 1.2.1 as + * expected nodes running is 2 and this test will create 2 clusters with one node each. + * @see org.elasticsearch.test.ElasticsearchIntegrationTest#ensureClusterSizeConsistency() + * TODO Reactivate when it will be possible to set the number of running nodes + */ +@Ignore public class GceDifferentTagsOneTagTest extends AbstractGceComputeServiceTest { @Override diff --git a/src/test/java/org/elasticsearch/discovery/gce/GceDifferentTagsTwoTagsTest.java b/src/test/java/org/elasticsearch/discovery/gce/GceDifferentTagsTwoTagsTest.java index d414b02409f..cbdd9c5542b 100644 --- a/src/test/java/org/elasticsearch/discovery/gce/GceDifferentTagsTwoTagsTest.java +++ b/src/test/java/org/elasticsearch/discovery/gce/GceDifferentTagsTwoTagsTest.java @@ -21,7 +21,15 @@ package org.elasticsearch.discovery.gce; import org.elasticsearch.common.settings.ImmutableSettings; import org.elasticsearch.common.settings.Settings; +import org.junit.Ignore; +/** + * We need to ignore this test from elasticsearch version 1.2.1 as + * expected nodes running is 2 and this test will create 2 clusters with one node each. + * @see org.elasticsearch.test.ElasticsearchIntegrationTest#ensureClusterSizeConsistency() + * TODO Reactivate when it will be possible to set the number of running nodes + */ +@Ignore public class GceDifferentTagsTwoTagsTest extends AbstractGceComputeServiceTest { @Override From 524ebda2570799f28266144f34bcd7ddb21b0191 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Tue, 5 Aug 2014 14:16:58 +0200 Subject: [PATCH 53/96] Docs: make the welcome page more obvious Closes #31. (cherry picked from commit ab3cde3) --- .gitignore | 1 + README.md | 32 +- dev-tools/build_release.py | 722 ------------------------------------- dev-tools/release.py | 134 +++++++ dev-tools/upload-s3.py | 67 ---- 5 files changed, 155 insertions(+), 801 deletions(-) delete mode 100755 dev-tools/build_release.py create mode 100644 dev-tools/release.py delete mode 100644 dev-tools/upload-s3.py diff --git a/.gitignore b/.gitignore index 87879ffa87d..38d7701b94d 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ /.classpath *.vmoptions .local-execution-hints.log +plugin_tools diff --git a/README.md b/README.md index 77ec97e2e35..ce7619fca52 100644 --- a/README.md +++ b/README.md @@ -3,22 +3,30 @@ Google Compute Engine Cloud Plugin for Elasticsearch The GCE Cloud plugin allows to use GCE API for the unicast discovery mechanism. -In order to install the plugin, simply run: `bin/plugin -install elasticsearch/elasticsearch-cloud-gce/2.1.1`. +In order to install the plugin, run: -* For master elasticsearch versions, look at [master branch](https://github.com/elasticsearch/elasticsearch-cloud-gce/tree/master). -* For 1.3.x elasticsearch versions, look at [es-1.3 branch](https://github.com/elasticsearch/elasticsearch-cloud-gce/tree/es-1.3). -* For 1.2.x elasticsearch versions, look at [es-1.2 branch](https://github.com/elasticsearch/elasticsearch-cloud-gce/tree/es-1.2). -* For 1.1.x elasticsearch versions, look at [es-1.1 branch](https://github.com/elasticsearch/elasticsearch-cloud-gce/tree/es-1.1). -* For 1.0.x elasticsearch versions, look at [es-1.0 branch](https://github.com/elasticsearch/elasticsearch-cloud-gce/tree/es-1.0). -* For 0.90.x elasticsearch versions, look at [es-0.90 branch](https://github.com/elasticsearch/elasticsearch-cloud-gce/tree/es-0.90). +```sh +bin/plugin -install elasticsearch/elasticsearch-cloud-gce/2.2.0 +``` -| GCE Cloud Plugin | elasticsearch | gce api | Release date | -|----------------------------|---------------------|--------------|:------------:| -| 3.0.0-SNAPSHOT | master | 1.18.0-rc | XXXX-XX-XX | +You need to install a version matching your Elasticsearch version: -Please read documentation relative to the version you are using: +| Elasticsearch | Azure Cloud Plugin| Docs | +|------------------------|-------------------|------------------------------------------------------------------------------------------------------------------------------------| +| master | Build from source | See below | +| es-1.3 | Build from source | [2.3.0-SNAPSHOT](https://github.com/elasticsearch/elasticsearch-cloud-gce/tree/es-1.3/#google-compute-engine-cloud-plugin-for-elasticsearch)| +| es-1.2 | 2.2.0 | [2.2.0](https://github.com/elasticsearch/elasticsearch-cloud-gce/tree/v2.2.0/#google-compute-engine-cloud-plugin-for-elasticsearch)| +| es-1.1 | 2.1.2 | [2.1.2](https://github.com/elasticsearch/elasticsearch-cloud-gce/tree/v2.1.2/#google-compute-engine-cloud-plugin-for-elasticsearch)| +| es-1.0 | 2.0.1 | [2.0.1](https://github.com/elasticsearch/elasticsearch-cloud-gce/tree/v2.0.1/#google-compute-engine-cloud-plugin-for-elasticsearch)| +| es-0.90 | 1.3.0 | [1.3.0](https://github.com/elasticsearch/elasticsearch-cloud-gce/tree/v1.3.0/#google-compute-engine-cloud-plugin-for-elasticsearch)| -* [3.0.0-SNAPSHOT](https://github.com/elasticsearch/elasticsearch-cloud-gce/blob/master/README.md) +To build a `SNAPSHOT` version, you need to build it with Maven: + +```bash +mvn clean install +plugin --install cloud-azure \ + --url file:target/releases/elasticsearch-cloud-gce-X.X.X-SNAPSHOT.zip +``` Google Compute Engine Virtual Machine Discovery diff --git a/dev-tools/build_release.py b/dev-tools/build_release.py deleted file mode 100755 index db8345440c7..00000000000 --- a/dev-tools/build_release.py +++ /dev/null @@ -1,722 +0,0 @@ -# Licensed to Elasticsearch under one or more contributor -# license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright -# ownership. Elasticsearch licenses this file to you under -# the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on -# an 'AS IS' BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -# either express or implied. See the License for the specific -# language governing permissions and limitations under the License. - -import re -import tempfile -import shutil -import os -import datetime -import argparse -import github3 -import smtplib - -from email.mime.multipart import MIMEMultipart -from email.mime.text import MIMEText - -from os.path import dirname, abspath - -""" - This tool builds a release from the a given elasticsearch plugin branch. - In order to execute it go in the top level directory and run: - $ python3 dev_tools/build_release.py --branch master --publish --remote origin - - By default this script runs in 'dry' mode which essentially simulates a release. If the - '--publish' option is set the actual release is done. - If not in 'dry' mode, a mail will be automatically sent to the mailing list. - You can disable it with the option '--disable_mail' - - $ python3 dev_tools/build_release.py --publish --remote origin --disable_mail - - The script takes over almost all - steps necessary for a release from a high level point of view it does the following things: - - - run prerequisite checks ie. check for S3 credentials available as env variables - - detect the version to release from the specified branch (--branch) or the current branch - - creates a release branch & updates pom.xml and README.md to point to a release version rather than a snapshot - - builds the artifacts - - commits the new version and merges the release branch into the source branch - - creates a tag and pushes the commit to the specified origin (--remote) - - publishes the releases to sonatype and S3 - - send a mail based on github issues fixed by this version - -Once it's done it will print all the remaining steps. - - Prerequisites: - - Python 3k for script execution - - Boto for S3 Upload ($ apt-get install python-boto or pip-3.3 install boto) - - github3 module (pip-3.3 install github3.py) - - S3 keys exported via ENV Variables (AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY) - - GITHUB (login/password) or key exported via ENV Variables (GITHUB_LOGIN, GITHUB_PASSWORD or GITHUB_KEY) - (see https://github.com/settings/applications#personal-access-tokens) - Optional: default to no authentication - - SMTP_HOST - Optional: default to localhost - - MAIL_SENDER - Optional: default to 'david@pilato.fr': must be authorized to send emails to elasticsearch mailing list - - MAIL_TO - Optional: default to 'elasticsearch@googlegroups.com' -""" -env = os.environ - -LOG = env.get('ES_RELEASE_LOG', '/tmp/elasticsearch_release.log') -ROOT_DIR = os.path.join(abspath(dirname(__file__)), '../') -README_FILE = ROOT_DIR + 'README.md' -POM_FILE = ROOT_DIR + 'pom.xml' - -def log(msg): - log_plain('\n%s' % msg) - -def log_plain(msg): - f = open(LOG, mode='ab') - f.write(msg.encode('utf-8')) - f.close() - -def run(command, quiet=False): - log('%s: RUN: %s\n' % (datetime.datetime.now(), command)) - if os.system('%s >> %s 2>&1' % (command, LOG)): - msg = ' FAILED: %s [see log %s]' % (command, LOG) - if not quiet: - print(msg) - raise RuntimeError(msg) - -try: - JAVA_HOME = env['JAVA_HOME'] -except KeyError: - raise RuntimeError(""" - Please set JAVA_HOME in the env before running release tool - On OSX use: export JAVA_HOME=`/usr/libexec/java_home -v '1.6*'`""") - -try: - MVN='mvn' - # make sure mvn3 is used if mvn3 is available - # some systems use maven 2 as default - run('mvn3 --version', quiet=True) - MVN='mvn3' -except RuntimeError: - pass - - -def java_exe(): - path = JAVA_HOME - return 'export JAVA_HOME="%s" PATH="%s/bin:$PATH" JAVACMD="%s/bin/java"' % (path, path, path) - -# Returns the hash of the current git HEAD revision -def get_head_hash(): - return os.popen(' git rev-parse --verify HEAD 2>&1').read().strip() - -# Returns the hash of the given tag revision -def get_tag_hash(tag): - return os.popen('git show-ref --tags %s --hash 2>&1' % (tag)).read().strip() - -# Returns the name of the current branch -def get_current_branch(): - return os.popen('git rev-parse --abbrev-ref HEAD 2>&1').read().strip() - -# Utility that returns the name of the release branch for a given version -def release_branch(version): - return 'release_branch_%s' % version - -# runs get fetch on the given remote -def fetch(remote): - run('git fetch %s' % remote) - -# Creates a new release branch from the given source branch -# and rebases the source branch from the remote before creating -# the release branch. Note: This fails if the source branch -# doesn't exist on the provided remote. -def create_release_branch(remote, src_branch, release): - run('git checkout %s' % src_branch) - run('git pull --rebase %s %s' % (remote, src_branch)) - run('git checkout -b %s' % (release_branch(release))) - - -# Reads the given file and applies the -# callback to it. If the callback changed -# a line the given file is replaced with -# the modified input. -def process_file(file_path, line_callback): - fh, abs_path = tempfile.mkstemp() - modified = False - with open(abs_path,'w', encoding='utf-8') as new_file: - with open(file_path, encoding='utf-8') as old_file: - for line in old_file: - new_line = line_callback(line) - modified = modified or (new_line != line) - new_file.write(new_line) - os.close(fh) - if modified: - #Remove original file - os.remove(file_path) - #Move new file - shutil.move(abs_path, file_path) - return True - else: - # nothing to do - just remove the tmp file - os.remove(abs_path) - return False - -# Guess the next snapshot version number (increment second digit) -def guess_snapshot(version): - digits=list(map(int, re.findall(r'\d+', version))) - source='%s.%s' % (digits[0], digits[1]) - destination='%s.%s' % (digits[0], digits[1]+1) - return version.replace(source, destination) - -# Moves the pom.xml file from a snapshot to a release -def remove_maven_snapshot(pom, release): - pattern = '%s-SNAPSHOT' % release - replacement = '%s' % release - def callback(line): - return line.replace(pattern, replacement) - process_file(pom, callback) - -# Moves the README.md file from a snapshot to a release -def remove_version_snapshot(readme_file, release): - pattern = '%s-SNAPSHOT' % release - replacement = '%s ' % release - def callback(line): - return line.replace(pattern, replacement) - process_file(readme_file, callback) - -# Moves the pom.xml file to the next snapshot -def add_maven_snapshot(pom, release, snapshot): - pattern = '%s' % release - replacement = '%s-SNAPSHOT' % snapshot - def callback(line): - return line.replace(pattern, replacement) - process_file(pom, callback) - -# Add in README.md file the next snapshot -def add_version_snapshot(readme_file, release, snapshot): - pattern = '| %s ' % release - replacement = '| %s-SNAPSHOT' % snapshot - def callback(line): - # If we find pattern, we copy the line and replace its content - if line.find(pattern) >= 0: - return line.replace(pattern, replacement).replace('%s' % (datetime.datetime.now().strftime("%Y-%m-%d")), - 'XXXX-XX-XX')+line - else: - return line - process_file(readme_file, callback) - -# Moves the README.md file from a snapshot to a release (documentation link) -def remove_documentation_snapshot(readme_file, repo_url, release, branch): - pattern = '* [%s-SNAPSHOT](%sblob/%s/README.md)' % (release, repo_url, branch) - replacement = '* [%s](%sblob/v%s/README.md)' % (release, repo_url, release) - def callback(line): - # If we find pattern, we replace its content - if line.find(pattern) >= 0: - return line.replace(pattern, replacement) - else: - return line - process_file(readme_file, callback) - -# Add in README.markdown file the documentation for the next version -def add_documentation_snapshot(readme_file, repo_url, release, snapshot, branch): - pattern = '* [%s](%sblob/v%s/README.md)' % (release, repo_url, release) - replacement = '* [%s-SNAPSHOT](%sblob/%s/README.md)' % (snapshot, repo_url, branch) - def callback(line): - # If we find pattern, we copy the line and replace its content - if line.find(pattern) >= 0: - return line.replace(pattern, replacement)+line - else: - return line - process_file(readme_file, callback) - -# Set release date in README.md file -def set_date(readme_file): - pattern = 'XXXX-XX-XX' - replacement = '%s' % (datetime.datetime.now().strftime("%Y-%m-%d")) - def callback(line): - return line.replace(pattern, replacement) - process_file(readme_file, callback) - -# Update installation instructions in README.md file -def set_install_instructions(readme_file, artifact_name, release): - pattern = '`bin/plugin -install elasticsearch/%s/.+`' % artifact_name - replacement = '`bin/plugin -install elasticsearch/%s/%s`' % (artifact_name, release) - def callback(line): - return re.sub(pattern, replacement, line) - process_file(readme_file, callback) - - -# Stages the given files for the next git commit -def add_pending_files(*files): - for file in files: - run('git add %s' % file) - -# Executes a git commit with 'release [version]' as the commit message -def commit_release(artifact_id, release): - run('git commit -m "prepare release %s-%s"' % (artifact_id, release)) - -def commit_snapshot(): - run('git commit -m "prepare for next development iteration"') - -def tag_release(release): - run('git tag -a v%s -m "Tag release version %s"' % (release, release)) - -def run_mvn(*cmd): - for c in cmd: - run('%s; %s -f %s %s' % (java_exe(), MVN, POM_FILE, c)) - -def build_release(run_tests=False, dry_run=True): - target = 'deploy' - if dry_run: - target = 'package' - if run_tests: - run_mvn('clean test') - run_mvn('clean %s -DskipTests' %(target)) - -# Checks the pom.xml for the release version. 2.0.0-SNAPSHOT -# This method fails if the pom file has no SNAPSHOT version set ie. -# if the version is already on a release version we fail. -# Returns the next version string ie. 0.90.7 -def find_release_version(src_branch): - run('git checkout %s' % src_branch) - with open(POM_FILE, encoding='utf-8') as file: - for line in file: - match = re.search(r'(.+)-SNAPSHOT', line) - if match: - return match.group(1) - raise RuntimeError('Could not find release version in branch %s' % src_branch) - -# extract a value from pom.xml -def find_from_pom(tag): - with open(POM_FILE, encoding='utf-8') as file: - for line in file: - match = re.search(r'<%s>(.+)' % (tag, tag), line) - if match: - return match.group(1) - raise RuntimeError('Could not find <%s> in pom.xml file' % (tag)) - -def get_artifacts(artifact_id, release): - artifact_path = ROOT_DIR + 'target/releases/%s-%s.zip' % (artifact_id, release) - print(' Path %s' % (artifact_path)) - if not os.path.isfile(artifact_path): - raise RuntimeError('Could not find required artifact at %s' % (artifact_path)) - return artifact_path - -# Generates sha1 for a file -# and returns the checksum files as well -# as the given files in a list -def generate_checksums(release_file): - res = [] - directory = os.path.dirname(release_file) - file = os.path.basename(release_file) - checksum_file = '%s.sha1.txt' % file - - if os.system('cd %s; shasum %s > %s' % (directory, file, checksum_file)): - raise RuntimeError('Failed to generate checksum for file %s' % release_file) - res = res + [os.path.join(directory, checksum_file), release_file] - return res - -def git_merge(src_branch, release_version): - run('git checkout %s' % src_branch) - run('git merge %s' % release_branch(release_version)) - -def git_push(remote, src_branch, release_version, dry_run): - if not dry_run: - run('git push %s %s' % (remote, src_branch)) # push the commit - run('git push %s v%s' % (remote, release_version)) # push the tag - else: - print(' dryrun [True] -- skipping push to remote %s' % remote) - -def publish_artifacts(artifacts, base='elasticsearch/elasticsearch', dry_run=True): - location = os.path.dirname(os.path.realpath(__file__)) - for artifact in artifacts: - if dry_run: - print('Skip Uploading %s to Amazon S3 in %s' % (artifact, base)) - else: - print('Uploading %s to Amazon S3' % artifact) - # requires boto to be installed but it is not available on python3k yet so we use a dedicated tool - run('python %s/upload-s3.py --file %s --path %s' % (location, os.path.abspath(artifact), base)) - - -################# -## -## -## Email and Github Management -## -## -################# -def format_issues_plain(issues, title='Fix'): - response = "" - - if len(issues) > 0: - response += '%s:\n' % title - for issue in issues: - response += ' * [%s] - %s (%s)\n' % (issue.number, issue.title, issue.html_url) - - return response - -def format_issues_html(issues, title='Fix'): - response = "" - - if len(issues) > 0: - response += '

%s

\n
    \n' % title - for issue in issues: - response += '
  • [%s] - %s\n' % (issue.html_url, issue.number, issue.title) - response += '
\n' - - return response - -def get_github_repository(reponame, - login=env.get('GITHUB_LOGIN', None), - password=env.get('GITHUB_PASSWORD', None), - key=env.get('GITHUB_KEY', None)): - if login: - g = github3.login(login, password) - elif key: - g = github3.login(token=key) - else: - g = github3.GitHub() - - return g.repository("elasticsearch", reponame) - -# Check if there are some remaining open issues and fails -def check_opened_issues(version, repository, reponame): - opened_issues = [i for i in repository.iter_issues(state='open', labels='%s' % version)] - if len(opened_issues)>0: - raise NameError('Some issues [%s] are still opened. Check https://github.com/elasticsearch/%s/issues?labels=%s&state=open' - % (len(opened_issues), reponame, version)) - -# List issues from github: can be done anonymously if you don't -# exceed a given number of github API calls per day -# Check if there are some remaining open issues and fails -def list_issues(version, - repository, - severity='bug'): - issues = [i for i in repository.iter_issues(state='closed', labels='%s,%s' % (severity, version))] - return issues - -# Get issues from github and generates a Plain/HTML Multipart email -# And send it if dry_run=False -def prepare_email(artifact_id, release_version, repository, - artifact_name, artifact_description, project_url, - severity_labels_bug='bug', - severity_labels_update='update', - severity_labels_new='new', - severity_labels_doc='doc'): - - ## Get bugs from github - issues_bug = list_issues(release_version, repository, severity=severity_labels_bug) - issues_update = list_issues(release_version, repository, severity=severity_labels_update) - issues_new = list_issues(release_version, repository, severity=severity_labels_new) - issues_doc = list_issues(release_version, repository, severity=severity_labels_doc) - - ## Format content to plain text - plain_issues_bug = format_issues_plain(issues_bug, 'Fix') - plain_issues_update = format_issues_plain(issues_update, 'Update') - plain_issues_new = format_issues_plain(issues_new, 'New') - plain_issues_doc = format_issues_plain(issues_doc, 'Doc') - - ## Format content to html - html_issues_bug = format_issues_html(issues_bug, 'Fix') - html_issues_update = format_issues_html(issues_update, 'Update') - html_issues_new = format_issues_html(issues_new, 'New') - html_issues_doc = format_issues_html(issues_doc, 'Doc') - - if len(issues_bug)+len(issues_update)+len(issues_new)+len(issues_doc) > 0: - plain_empty_message = "" - html_empty_message = "" - - else: - plain_empty_message = "No issue listed for this release" - html_empty_message = "

No issue listed for this release

" - - msg = MIMEMultipart('alternative') - msg['Subject'] = '[ANN] %s %s released' % (artifact_name, release_version) - text = """ -Heya, - - -We are pleased to announce the release of the %(artifact_name)s, version %(release_version)s. - -%(artifact_description)s. - -%(project_url)s - -Release Notes - %(artifact_id)s - Version %(release_version)s - -%(empty_message)s -%(issues_bug)s -%(issues_update)s -%(issues_new)s -%(issues_doc)s - -Issues, Pull requests, Feature requests are warmly welcome on %(artifact_id)s project repository: %(project_url)s -For questions or comments around this plugin, feel free to use elasticsearch mailing list: https://groups.google.com/forum/#!forum/elasticsearch - -Enjoy, - --The Elasticsearch team -""" % {'release_version': release_version, - 'artifact_id': artifact_id, - 'artifact_name': artifact_name, - 'artifact_description': artifact_description, - 'project_url': project_url, - 'empty_message': plain_empty_message, - 'issues_bug': plain_issues_bug, - 'issues_update': plain_issues_update, - 'issues_new': plain_issues_new, - 'issues_doc': plain_issues_doc} - - html = """ - - -

Heya,

- -

We are pleased to announce the release of the %(artifact_name)s, version %(release_version)s

- -
%(artifact_description)s.
- -

Release Notes - Version %(release_version)s

-%(empty_message)s -%(issues_bug)s -%(issues_update)s -%(issues_new)s -%(issues_doc)s - -

Issues, Pull requests, Feature requests are warmly welcome on -%(artifact_id)s project repository!

-

For questions or comments around this plugin, feel free to use elasticsearch -mailing list!

- -

Enjoy,

- -

- The Elasticsearch team

- -""" % {'release_version': release_version, - 'artifact_id': artifact_id, - 'artifact_name': artifact_name, - 'artifact_description': artifact_description, - 'project_url': project_url, - 'empty_message': html_empty_message, - 'issues_bug': html_issues_bug, - 'issues_update': html_issues_update, - 'issues_new': html_issues_new, - 'issues_doc': html_issues_doc} - - # Record the MIME types of both parts - text/plain and text/html. - part1 = MIMEText(text, 'plain') - part2 = MIMEText(html, 'html') - - # Attach parts into message container. - # According to RFC 2046, the last part of a multipart message, in this case - # the HTML message, is best and preferred. - msg.attach(part1) - msg.attach(part2) - - return msg - -def send_email(msg, - dry_run=True, - mail=True, - sender=env.get('MAIL_SENDER'), - to=env.get('MAIL_TO', 'elasticsearch@googlegroups.com'), - smtp_server=env.get('SMTP_SERVER', 'localhost')): - msg['From'] = 'Elasticsearch Team <%s>' % sender - msg['To'] = 'Elasticsearch Mailing List <%s>' % to - # save mail on disk - with open(ROOT_DIR+'target/email.txt', 'w') as email_file: - email_file.write(msg.as_string()) - if mail and not dry_run: - s = smtplib.SMTP(smtp_server, 25) - s.sendmail(sender, to, msg.as_string()) - s.quit() - else: - print('generated email: open %starget/email.txt' % ROOT_DIR) - -def print_sonatype_notice(): - settings = os.path.join(os.path.expanduser('~'), '.m2/settings.xml') - if os.path.isfile(settings): - with open(settings, encoding='utf-8') as settings_file: - for line in settings_file: - if line.strip() == 'sonatype-nexus-snapshots': - # moving out - we found the indicator no need to print the warning - return - print(""" - NOTE: No sonatype settings detected, make sure you have configured - your sonatype credentials in '~/.m2/settings.xml': - - - ... - - - sonatype-nexus-snapshots - your-jira-id - your-jira-pwd - - - sonatype-nexus-staging - your-jira-id - your-jira-pwd - - - ... - - """) - -def check_s3_credentials(): - if not env.get('AWS_ACCESS_KEY_ID', None) or not env.get('AWS_SECRET_ACCESS_KEY', None): - raise RuntimeError('Could not find "AWS_ACCESS_KEY_ID" / "AWS_SECRET_ACCESS_KEY" in the env variables please export in order to upload to S3') - -def check_github_credentials(): - if not env.get('GITHUB_KEY', None) and not env.get('GITHUB_LOGIN', None): - log('WARN: Could not find "GITHUB_LOGIN" / "GITHUB_PASSWORD" or "GITHUB_KEY" in the env variables. You could need it.') - -def check_email_settings(): - if not env.get('MAIL_SENDER', None): - raise RuntimeError('Could not find "MAIL_SENDER"') - -# we print a notice if we can not find the relevant infos in the ~/.m2/settings.xml -print_sonatype_notice() - -if __name__ == '__main__': - parser = argparse.ArgumentParser(description='Builds and publishes a Elasticsearch Plugin Release') - parser.add_argument('--branch', '-b', metavar='master', default=get_current_branch(), - help='The branch to release from. Defaults to the current branch.') - parser.add_argument('--skiptests', '-t', dest='tests', action='store_false', - help='Skips tests before release. Tests are run by default.') - parser.set_defaults(tests=True) - parser.add_argument('--remote', '-r', metavar='origin', default='origin', - help='The remote to push the release commit and tag to. Default is [origin]') - parser.add_argument('--publish', '-p', dest='dryrun', action='store_false', - help='Publishes the release. Disable by default.') - parser.add_argument('--disable_mail', '-dm', dest='mail', action='store_false', - help='Do not send a release email. Email is sent by default.') - - parser.set_defaults(dryrun=True) - parser.set_defaults(mail=True) - args = parser.parse_args() - - src_branch = args.branch - remote = args.remote - run_tests = args.tests - dry_run = args.dryrun - mail = args.mail - - if not dry_run: - check_s3_credentials() - print('WARNING: dryrun is set to "false" - this will push and publish the release') - if mail: - check_email_settings() - print('An email to %s will be sent after the release' - % env.get('MAIL_TO', 'elasticsearch@googlegroups.com')) - input('Press Enter to continue...') - - check_github_credentials() - - print(''.join(['-' for _ in range(80)])) - print('Preparing Release from branch [%s] running tests: [%s] dryrun: [%s]' % (src_branch, run_tests, dry_run)) - print(' JAVA_HOME is [%s]' % JAVA_HOME) - print(' Running with maven command: [%s] ' % (MVN)) - - release_version = find_release_version(src_branch) - artifact_id = find_from_pom('artifactId') - artifact_name = find_from_pom('name') - artifact_description = find_from_pom('description') - project_url = find_from_pom('url') - elasticsearch_version = find_from_pom('elasticsearch.version') - print(' Artifact Id: [%s]' % artifact_id) - print(' Release version: [%s]' % release_version) - print(' Elasticsearch: [%s]' % elasticsearch_version) - if elasticsearch_version.find('-SNAPSHOT') != -1: - raise RuntimeError('Can not release with a SNAPSHOT elasticsearch dependency: %s' % elasticsearch_version) - - # extract snapshot - default_snapshot_version = guess_snapshot(release_version) - snapshot_version = input('Enter next snapshot version [%s]:' % default_snapshot_version) - snapshot_version = snapshot_version or default_snapshot_version - - print(' Next version: [%s-SNAPSHOT]' % snapshot_version) - print(' Artifact Name: [%s]' % artifact_name) - print(' Artifact Description: [%s]' % artifact_description) - print(' Project URL: [%s]' % project_url) - - if not dry_run: - smoke_test_version = release_version - head_hash = get_head_hash() - run_mvn('clean') # clean the env! - create_release_branch(remote, src_branch, release_version) - print(' Created release branch [%s]' % (release_branch(release_version))) - success = False - try: - pending_files = [POM_FILE, README_FILE] - remove_maven_snapshot(POM_FILE, release_version) - remove_documentation_snapshot(README_FILE, project_url, release_version, src_branch) - remove_version_snapshot(README_FILE, release_version) - set_date(README_FILE) - set_install_instructions(README_FILE, artifact_id, release_version) - print(' Done removing snapshot version') - add_pending_files(*pending_files) # expects var args use * to expand - commit_release(artifact_id, release_version) - print(' Committed release version [%s]' % release_version) - print(''.join(['-' for _ in range(80)])) - print('Building Release candidate') - input('Press Enter to continue...') - print(' Checking github issues') - repository = get_github_repository(artifact_id) - check_opened_issues(release_version, repository, artifact_id) - if not dry_run: - print(' Running maven builds now and publish to sonatype - run-tests [%s]' % run_tests) - else: - print(' Running maven builds now run-tests [%s]' % run_tests) - build_release(run_tests=run_tests, dry_run=dry_run) - artifact = get_artifacts(artifact_id, release_version) - artifact_and_checksums = generate_checksums(artifact) - print(''.join(['-' for _ in range(80)])) - - print('Finish Release -- dry_run: %s' % dry_run) - input('Press Enter to continue...') - print(' merge release branch') - git_merge(src_branch, release_version) - print(' tag') - tag_release(release_version) - - add_maven_snapshot(POM_FILE, release_version, snapshot_version) - add_version_snapshot(README_FILE, release_version, snapshot_version) - add_documentation_snapshot(README_FILE, project_url, release_version, snapshot_version, src_branch) - add_pending_files(*pending_files) - commit_snapshot() - - print(' push to %s %s -- dry_run: %s' % (remote, src_branch, dry_run)) - git_push(remote, src_branch, release_version, dry_run) - print(' publish artifacts to S3 -- dry_run: %s' % dry_run) - publish_artifacts(artifact_and_checksums, base='elasticsearch/%s' % (artifact_id) , dry_run=dry_run) - print(' preparing email (from github issues)') - msg = prepare_email(artifact_id, release_version, repository, artifact_name, artifact_description, project_url) - print(' sending email -- dry_run: %s, mail: %s' % (dry_run, mail)) - send_email(msg, dry_run=dry_run, mail=mail) - - pending_msg = """ -Release successful pending steps: - * close and release sonatype repo: https://oss.sonatype.org/ - * check if the release is there https://oss.sonatype.org/content/repositories/releases/org/elasticsearch/%(artifact_id)s/%(version)s - * tweet about the release -""" - print(pending_msg % {'version': release_version, - 'artifact_id': artifact_id, - 'project_url': project_url}) - success = True - finally: - if not success: - run('git reset --hard HEAD') - run('git checkout %s' % src_branch) - elif dry_run: - print('End of dry_run') - input('Press Enter to reset changes...') - - run('git reset --hard %s' % head_hash) - run('git tag -d v%s' % release_version) - # we delete this one anyways - run('git branch -D %s' % (release_branch(release_version))) diff --git a/dev-tools/release.py b/dev-tools/release.py new file mode 100644 index 00000000000..edcc637d068 --- /dev/null +++ b/dev-tools/release.py @@ -0,0 +1,134 @@ +# Licensed to Elasticsearch under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on +# an 'AS IS' BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, +# either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + +import datetime +import os +import shutil +import sys +import time +import urllib +import urllib.request +import zipfile + +from os.path import dirname, abspath + +""" + This tool builds a release from the a given elasticsearch plugin branch. + + It is basically a wrapper on top of launch_release.py which: + + - tries to get a more recent version of launch_release.py in ... + - download it if needed + - launch it passing all arguments to it, like: + + $ python3 dev_tools/release.py --branch master --publish --remote origin + + Important options: + + # Dry run + $ python3 dev_tools/release.py + + # Dry run without tests + python3 dev_tools/release.py --skiptests + + # Release, publish artifacts and announce + $ python3 dev_tools/release.py --publish + + See full documentation in launch_release.py +""" +env = os.environ + +# Change this if the source repository for your scripts is at a different location +SOURCE_REPO = 'elasticsearch/elasticsearch-plugins-script' +# We define that we should download again the script after 1 days +SCRIPT_OBSOLETE_DAYS = 1 +# We ignore in master.zip file the following files +IGNORED_FILES = ['.gitignore', 'README.md'] + + +ROOT_DIR = abspath(os.path.join(abspath(dirname(__file__)), '../')) +TARGET_TOOLS_DIR = ROOT_DIR + '/plugin_tools' +DEV_TOOLS_DIR = ROOT_DIR + '/dev-tools' +BUILD_RELEASE_FILENAME = 'release.zip' +BUILD_RELEASE_FILE = TARGET_TOOLS_DIR + '/' + BUILD_RELEASE_FILENAME +SOURCE_URL = 'https://github.com/%s/archive/master.zip' % SOURCE_REPO + +# Download a recent version of the release plugin tool +try: + os.mkdir(TARGET_TOOLS_DIR) + print('directory %s created' % TARGET_TOOLS_DIR) +except FileExistsError: + pass + + +try: + # we check latest update. If we ran an update recently, we + # are not going to check it again + download = True + + try: + last_download_time = datetime.datetime.fromtimestamp(os.path.getmtime(BUILD_RELEASE_FILE)) + if (datetime.datetime.now()-last_download_time).days < SCRIPT_OBSOLETE_DAYS: + download = False + except FileNotFoundError: + pass + + if download: + urllib.request.urlretrieve(SOURCE_URL, BUILD_RELEASE_FILE) + with zipfile.ZipFile(BUILD_RELEASE_FILE) as myzip: + for member in myzip.infolist(): + filename = os.path.basename(member.filename) + # skip directories + if not filename: + continue + if filename in IGNORED_FILES: + continue + + # copy file (taken from zipfile's extract) + source = myzip.open(member.filename) + target = open(os.path.join(TARGET_TOOLS_DIR, filename), "wb") + with source, target: + shutil.copyfileobj(source, target) + # We keep the original date + date_time = time.mktime(member.date_time + (0, 0, -1)) + os.utime(os.path.join(TARGET_TOOLS_DIR, filename), (date_time, date_time)) + print('plugin-tools updated from %s' % SOURCE_URL) +except urllib.error.HTTPError: + pass + + +# Let see if we need to update the release.py script itself +source_time = os.path.getmtime(TARGET_TOOLS_DIR + '/release.py') +repo_time = os.path.getmtime(DEV_TOOLS_DIR + '/release.py') +if source_time > repo_time: + input('release.py needs an update. Press a key to update it...') + shutil.copyfile(TARGET_TOOLS_DIR + '/release.py', DEV_TOOLS_DIR + '/release.py') + +# We can launch the build process +try: + PYTHON = 'python' + # make sure python3 is used if python3 is available + # some systems use python 2 as default + os.system('python3 --version > /dev/null 2>&1') + PYTHON = 'python3' +except RuntimeError: + pass + +release_args = '' +for x in range(1, len(sys.argv)): + release_args += ' ' + sys.argv[x] + +os.system('%s %s/build_release.py %s' % (PYTHON, TARGET_TOOLS_DIR, release_args)) diff --git a/dev-tools/upload-s3.py b/dev-tools/upload-s3.py deleted file mode 100644 index 95ea576e65c..00000000000 --- a/dev-tools/upload-s3.py +++ /dev/null @@ -1,67 +0,0 @@ -# Licensed to Elasticsearch under one or more contributor -# license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright -# ownership. Elasticsearch licenses this file to you under -# the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on -# an 'AS IS' BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -# either express or implied. See the License for the specific -# language governing permissions and limitations under the License. - -import os -import sys -import argparse -try: - import boto.s3 -except: - raise RuntimeError(""" - S3 upload requires boto to be installed - Use one of: - 'pip install -U boto' - 'apt-get install python-boto' - 'easy_install boto' - """) - -import boto.s3 - - -def list_buckets(conn): - return conn.get_all_buckets() - - -def upload_s3(conn, path, key, file, bucket): - print 'Uploading %s to Amazon S3 bucket %s/%s' % \ - (file, bucket, os.path.join(path, key)) - def percent_cb(complete, total): - sys.stdout.write('.') - sys.stdout.flush() - bucket = conn.create_bucket(bucket) - k = bucket.new_key(os.path.join(path, key)) - k.set_contents_from_filename(file, cb=percent_cb, num_cb=100) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser(description='Uploads files to Amazon S3') - parser.add_argument('--file', '-f', metavar='path to file', - help='the branch to release from', required=True) - parser.add_argument('--bucket', '-b', metavar='B42', default='download.elasticsearch.org', - help='The S3 Bucket to upload to') - parser.add_argument('--path', '-p', metavar='elasticsearch/elasticsearch', default='elasticsearch/elasticsearch', - help='The key path to use') - parser.add_argument('--key', '-k', metavar='key', default=None, - help='The key - uses the file name as default key') - args = parser.parse_args() - if args.key: - key = args.key - else: - key = os.path.basename(args.file) - - connection = boto.connect_s3() - upload_s3(connection, args.path, key, args.file, args.bucket); - From b45a05f2fb9448d2228f08f76c7b1d0915d8e2b5 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Tue, 5 Aug 2014 14:16:58 +0200 Subject: [PATCH 54/96] fix typo --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ce7619fca52..c828063f3e4 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ bin/plugin -install elasticsearch/elasticsearch-cloud-gce/2.2.0 You need to install a version matching your Elasticsearch version: -| Elasticsearch | Azure Cloud Plugin| Docs | +| Elasticsearch | GCE Cloud Plugin | Docs | |------------------------|-------------------|------------------------------------------------------------------------------------------------------------------------------------| | master | Build from source | See below | | es-1.3 | Build from source | [2.3.0-SNAPSHOT](https://github.com/elasticsearch/elasticsearch-cloud-gce/tree/es-1.3/#google-compute-engine-cloud-plugin-for-elasticsearch)| @@ -24,7 +24,7 @@ To build a `SNAPSHOT` version, you need to build it with Maven: ```bash mvn clean install -plugin --install cloud-azure \ +plugin --install cloud-gce \ --url file:target/releases/elasticsearch-cloud-gce-X.X.X-SNAPSHOT.zip ``` From 0db6959f71239a9b157956eb45ad7e8b5f79c60e Mon Sep 17 00:00:00 2001 From: "Simon (Big Chief) Morley" Date: Fri, 20 Jun 2014 00:27:29 +0100 Subject: [PATCH 55/96] Included notes about compute engine permissions Closes #25. Closes #26. --- README.md | 54 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 53 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index c828063f3e4..d4b7636b4f4 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ multicast environments). Here is a simple sample configuration: How to start (short story) -------------------------- -* Create Google Compute Engine instance +* Create Google Compute Engine instance (with compute rw permissions) * Install Elasticsearch * Install Google Compute Engine Cloud plugin * Modify `elasticsearch.yml` file @@ -113,6 +113,17 @@ gcloud compute ssh myesnode1 --zone europe-west1-a ssh -i ~/.ssh/google_compute_engine 192.158.29.199 ``` +*Note Regarding Service Account Permissions* + +It's important when creating an instance that the correct permissions are set. At a minimum, you must ensure you have: + +``` +service_account_scope=compute-rw +``` + +Failing to set this will result in unauthorized messages when starting Elasticsearch. +See [Machine Permissions](#machine-permissions). + Once connected, install Elasticsearch: ```sh @@ -313,6 +324,8 @@ gcutil setinstancemetadata myesnode1 \ Tips ---- +### Store project id locally + If you don't want to repeat the project id each time, you can save it in `~/.gcutil.flags` file using: ```sh @@ -325,6 +338,45 @@ gcutil getproject --project=es-cloud --cache_flag_values --project=es-cloud ``` +### Machine Permissions + +**Creating machines with gcutil** + +Ensure the following flags are set: + +```` +--service_account_scope=compute-rw +``` + +**Creating with console (web)** + +When creating an instance using the web portal, click **Show advanced options**. + +At the bottom of the page, under `PROJECT ACCESS`, choose `>> Compute >> Read Write`. + +**Creating with knife google** + +Set the service account scopes when creating the machine: + +``` +$ knife google server create www1 \ + -m n1-standard-1 \ + -I debian-7-wheezy-v20131120 \ + -Z us-central1-a \ + -i ~/.ssh/id_rsa \ + -x jdoe \ + --gce-service-account-scopes https://www.googleapis.com/auth/compute.full_control +``` + +Or, you may use the alias: + +``` + --gce-service-account-scopes compute-rw +``` + +If you have created a machine without the correct permissions, you will see `403 unauthorized` error messages. The only +way to alter these permissions is to delete the instance (NOT THE DISK). Then create another with the correct permissions. + License ------- From 8bd7e42bdf426eaa71ac81d41a22368fb99782ec Mon Sep 17 00:00:00 2001 From: "Simon (Big Chief) Morley" Date: Tue, 5 Aug 2014 16:07:35 +0200 Subject: [PATCH 56/96] Force Token URL to `http://metadata.google.internal/...` In certain environments (such as inside a docker container) 'metadata' does not resolve. Forcing URL to `http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/token` fix this. See https://developers.google.com/compute/docs/metadata#metadataserver Closes #28. Closes #30. --- .../cloud/gce/GceComputeServiceImpl.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/elasticsearch/cloud/gce/GceComputeServiceImpl.java b/src/main/java/org/elasticsearch/cloud/gce/GceComputeServiceImpl.java index 44f2cbc0c6b..5a08c5fefab 100644 --- a/src/main/java/org/elasticsearch/cloud/gce/GceComputeServiceImpl.java +++ b/src/main/java/org/elasticsearch/cloud/gce/GceComputeServiceImpl.java @@ -48,6 +48,11 @@ public class GceComputeServiceImpl extends AbstractLifecycleComponent instances() { try { @@ -98,7 +103,11 @@ public class GceComputeServiceImpl extends AbstractLifecycleComponent Date: Tue, 5 Aug 2014 16:07:35 +0200 Subject: [PATCH 57/96] Update to Lucene 4.9.0 Closes #32. --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 8ac04aa62cd..e647766302a 100644 --- a/pom.xml +++ b/pom.xml @@ -51,7 +51,7 @@ governing permissions and limitations under the License. --> 2.0.0-SNAPSHOT v1-rev15-1.18.0-rc 1.18.0-rc - 4.8.1 + 4.9.0 true onerror From 51c10c18bafc4538d4eccde8dad5eabc11c2c7f8 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Tue, 5 Aug 2014 16:07:35 +0200 Subject: [PATCH 58/96] Tests: refactor tests We need to simplify a bit our tests. Also, we need to mark as `@Ignore` tests as they break now due to some cluster controls after each test in elasticsearch test infra. See [org.elasticsearch.test.ElasticsearchIntegrationTest#ensureClusterSizeConsistency()](https://github.com/elasticsearch/elasticsearch/blob/v1.3.1/src/test/java/org/elasticsearch/test/ElasticsearchIntegrationTest.java#L974-L979) Closes #33. --- README.md | 25 +++ pom.xml | 2 +- .../cloud/gce/AbstractGceTest.java | 42 ++-- .../gce/AbstractGceComputeServiceTest.java | 72 ------- .../discovery/gce/GceComputeEngineTest.java | 182 ++++++++++++++++++ .../gce/GceDifferentTagsNoTagTest.java | 42 ---- .../gce/GceDifferentTagsOneTagTest.java | 51 ----- .../gce/GceDifferentTagsTwoTagsTest.java | 51 ----- .../discovery/gce/GceSameTagsNoTagTest.java | 42 ---- .../discovery/gce/GceSameTagsOneTagTest.java | 43 ----- .../discovery/gce/GceSameTagsTwoTagsTest.java | 43 ----- .../GceComputeServiceAbstractMock.java | 5 +- ...mputeServiceTwoNodesDifferentTagsMock.java | 2 +- ...GceComputeServiceTwoNodesSameTagsMock.java | 2 +- .../gce/itest/GceSimpleITest.java | 2 +- src/test/resources/elasticsearch.yml | 27 --- 16 files changed, 243 insertions(+), 390 deletions(-) delete mode 100644 src/test/java/org/elasticsearch/discovery/gce/AbstractGceComputeServiceTest.java create mode 100644 src/test/java/org/elasticsearch/discovery/gce/GceComputeEngineTest.java delete mode 100644 src/test/java/org/elasticsearch/discovery/gce/GceDifferentTagsNoTagTest.java delete mode 100644 src/test/java/org/elasticsearch/discovery/gce/GceDifferentTagsOneTagTest.java delete mode 100644 src/test/java/org/elasticsearch/discovery/gce/GceDifferentTagsTwoTagsTest.java delete mode 100644 src/test/java/org/elasticsearch/discovery/gce/GceSameTagsNoTagTest.java delete mode 100644 src/test/java/org/elasticsearch/discovery/gce/GceSameTagsOneTagTest.java delete mode 100644 src/test/java/org/elasticsearch/discovery/gce/GceSameTagsTwoTagsTest.java rename src/test/java/org/elasticsearch/discovery/gce/{ => mock}/GceComputeServiceAbstractMock.java (94%) rename src/test/java/org/elasticsearch/discovery/gce/{ => mock}/GceComputeServiceTwoNodesDifferentTagsMock.java (96%) rename src/test/java/org/elasticsearch/discovery/gce/{ => mock}/GceComputeServiceTwoNodesSameTagsMock.java (96%) delete mode 100644 src/test/resources/elasticsearch.yml diff --git a/README.md b/README.md index d4b7636b4f4..aef88c1a391 100644 --- a/README.md +++ b/README.md @@ -377,6 +377,31 @@ Or, you may use the alias: If you have created a machine without the correct permissions, you will see `403 unauthorized` error messages. The only way to alter these permissions is to delete the instance (NOT THE DISK). Then create another with the correct permissions. + +Testing +======= + +Integrations tests in this plugin require working GCE configuration and therefore disabled by default. +To enable tests prepare a config file elasticsearch.yml with the following content: + +``` +cloud: + gce: + project_id: es-cloud + zone: europe-west1-a +discovery: + type: gce +``` + +Replaces `project_id` and `zone` with your settings. + +To run test: + +```sh +mvn -Dtests.gce=true -Dtests.config=/path/to/config/file/elasticsearch.yml clean test +``` + + License ------- diff --git a/pom.xml b/pom.xml index e647766302a..c8d1b8f1085 100644 --- a/pom.xml +++ b/pom.xml @@ -243,6 +243,7 @@ governing permissions and limitations under the License. --> ${tests.weekly} ${tests.slow} ${tests.gce} + ${tests.config} ${tests.awaitsfix} ${tests.slow} ${tests.timeoutSuite} @@ -252,7 +253,6 @@ governing permissions and limitations under the License. --> ${tests.client.ratio} false network - ${es.config} ${es.logger.level} true diff --git a/src/test/java/org/elasticsearch/cloud/gce/AbstractGceTest.java b/src/test/java/org/elasticsearch/cloud/gce/AbstractGceTest.java index 115775f7c6d..6388a2c03f9 100644 --- a/src/test/java/org/elasticsearch/cloud/gce/AbstractGceTest.java +++ b/src/test/java/org/elasticsearch/cloud/gce/AbstractGceTest.java @@ -20,6 +20,12 @@ package org.elasticsearch.cloud.gce; import com.carrotsearch.randomizedtesting.annotations.TestGroup; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.ImmutableSettings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.env.FailedToResolveConfigException; +import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.test.ElasticsearchIntegrationTest; import java.lang.annotation.Documented; @@ -34,20 +40,9 @@ public abstract class AbstractGceTest extends ElasticsearchIntegrationTest { /** * Annotation for tests that require GCE to run. GCE tests are disabled by default. - *

- * To enable test add -Dtests.gce=true -Des.config=/path/to/elasticsearch.yml - *

- * The elasticsearch.yml file should contain the following keys - * 

-      cloud:
-          gce:
-              project_id: es-cloud
-              zone: europe-west1-a
-      discovery:
-              type: gce
-     *
+ * See README file for details. */ - @Documented + @Documented @Inherited @Retention(RetentionPolicy.RUNTIME) @TestGroup(enabled = false, sysProperty = SYSPROP_GCE) @@ -58,4 +53,25 @@ public abstract class AbstractGceTest extends ElasticsearchIntegrationTest { */ public static final String SYSPROP_GCE = "tests.gce"; + @Override + protected Settings nodeSettings(int nodeOrdinal) { + ImmutableSettings.Builder settings = ImmutableSettings.builder() + .put(super.nodeSettings(nodeOrdinal)) + .put("plugins." + PluginsService.LOAD_PLUGIN_FROM_CLASSPATH, true); + + Environment environment = new Environment(); + + // if explicit, just load it and don't load from env + try { + if (Strings.hasText(System.getProperty("tests.config"))) { + settings.loadFromUrl(environment.resolveConfig(System.getProperty("tests.config"))); + } else { + fail("to run integration tests, you need to set -Dtest.gce=true and -Dtests.config=/path/to/elasticsearch.yml"); + } + } catch (FailedToResolveConfigException exception) { + fail("your test configuration file is incorrect: " + System.getProperty("tests.config")); + } + return settings.build(); + } + } diff --git a/src/test/java/org/elasticsearch/discovery/gce/AbstractGceComputeServiceTest.java b/src/test/java/org/elasticsearch/discovery/gce/AbstractGceComputeServiceTest.java deleted file mode 100644 index ac4d6d7817b..00000000000 --- a/src/test/java/org/elasticsearch/discovery/gce/AbstractGceComputeServiceTest.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.discovery.gce; - -import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; -import org.elasticsearch.common.settings.ImmutableSettings; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.plugins.PluginsService; -import org.elasticsearch.test.ElasticsearchIntegrationTest; -import org.junit.Test; - -import java.io.IOException; - -import static org.hamcrest.CoreMatchers.notNullValue; -import static org.hamcrest.Matchers.is; - -@ElasticsearchIntegrationTest.ClusterScope( - scope = ElasticsearchIntegrationTest.Scope.SUITE, - numDataNodes = 2, - numClientNodes = 0, - transportClientRatio = 0.0) -public abstract class AbstractGceComputeServiceTest extends ElasticsearchIntegrationTest { - /** - * Set the number of expected nodes in the current cluster - */ - protected abstract int getExpectedNodes(); - - public static int getPort(int nodeOrdinal) { - try { - return PropertiesHelper.getAsInt("plugin.port") - + nodeOrdinal * 10; - } catch (IOException e) { - } - - return -1; - } - - @Override - protected Settings nodeSettings(int nodeOrdinal) { - return ImmutableSettings.builder() - .put("transport.tcp.port", getPort(nodeOrdinal)) - .put("http.enabled", false) - .put("plugins." + PluginsService.LOAD_PLUGIN_FROM_CLASSPATH, true) - .put(super.nodeSettings(nodeOrdinal)) - .build(); - } - - @Test - public void testExpectedNumberOfNodes() { - NodesInfoResponse nodeInfos = client().admin().cluster().prepareNodesInfo().execute().actionGet(); - - assertThat(nodeInfos.getNodes(), notNullValue()); - assertThat(nodeInfos.getNodes().length, is(getExpectedNodes())); - } -} diff --git a/src/test/java/org/elasticsearch/discovery/gce/GceComputeEngineTest.java b/src/test/java/org/elasticsearch/discovery/gce/GceComputeEngineTest.java new file mode 100644 index 00000000000..be991f92b49 --- /dev/null +++ b/src/test/java/org/elasticsearch/discovery/gce/GceComputeEngineTest.java @@ -0,0 +1,182 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.discovery.gce; + +import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; +import org.elasticsearch.cloud.gce.GceComputeService; +import org.elasticsearch.common.collect.Lists; +import org.elasticsearch.common.settings.ImmutableSettings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.discovery.gce.mock.GceComputeServiceTwoNodesDifferentTagsMock; +import org.elasticsearch.discovery.gce.mock.GceComputeServiceTwoNodesSameTagsMock; +import org.elasticsearch.plugins.PluginsService; +import org.elasticsearch.test.ElasticsearchIntegrationTest; +import org.junit.Ignore; +import org.junit.Test; + +import java.io.IOException; + +import static org.hamcrest.Matchers.notNullValue; + +@ElasticsearchIntegrationTest.ClusterScope( + scope = ElasticsearchIntegrationTest.Scope.TEST, + numDataNodes = 0, + numClientNodes = 0, + transportClientRatio = 0.0) +public class GceComputeEngineTest extends ElasticsearchIntegrationTest { + + public static int getPort(int nodeOrdinal) { + try { + return PropertiesHelper.getAsInt("plugin.port") + + nodeOrdinal * 10; + } catch (IOException e) { + } + + return -1; + } + + protected void checkNumberOfNodes(int expected) { + NodesInfoResponse nodeInfos = client().admin().cluster().prepareNodesInfo().execute().actionGet(); + assertNotNull(nodeInfos); + assertNotNull(nodeInfos.getNodes()); + assertEquals(expected, nodeInfos.getNodes().length); + } + + protected Settings settingsBuilder(int nodeOrdinal, Class mock, Settings settings) { + ImmutableSettings.Builder builder = ImmutableSettings.settingsBuilder() + .put("discovery.type", "gce") + .put("cloud.gce.api.impl", mock) + // We need the network to make the mock working + .put("node.mode", "network") + // Make the tests run faster + .put("discovery.zen.join.timeout", "100ms") + .put("discovery.zen.ping.timeout", "10ms") + .put("discovery.initial_state_timeout", "300ms") + // We use a specific port for each node + .put("transport.tcp.port", getPort(nodeOrdinal)) + // We disable http + .put("http.enabled", false) + // We force plugin loading + .put("plugins." + PluginsService.LOAD_PLUGIN_FROM_CLASSPATH, true) + .put(settings) + .put(super.nodeSettings(nodeOrdinal)); + + return builder.build(); + } + + protected void startNode(int nodeOrdinal, Class mock, Settings settings) { + logger.info("--> start node #{}, mock [{}], settings [{}]", nodeOrdinal, mock.getSimpleName(), settings.getAsMap()); + internalCluster().startNode(settingsBuilder( + nodeOrdinal, + mock, + settings)); + assertThat(client().admin().cluster().prepareState().setMasterNodeTimeout("1s").execute().actionGet().getState().nodes().masterNodeId(), notNullValue()); + } + + @Test @Ignore + public void nodes_with_different_tags_and_no_tag_set() { + startNode(1, + GceComputeServiceTwoNodesDifferentTagsMock.class, + ImmutableSettings.EMPTY); + startNode(2, + GceComputeServiceTwoNodesDifferentTagsMock.class, + ImmutableSettings.EMPTY); + + // We expect having 2 nodes as part of the cluster, let's test that + checkNumberOfNodes(2); + } + + /** + * We need to ignore this test from elasticsearch version 1.2.1 as + * expected nodes running is 2 and this test will create 2 clusters with one node each. + * @see org.elasticsearch.test.ElasticsearchIntegrationTest#ensureClusterSizeConsistency() + * TODO Reactivate when it will be possible to set the number of running nodes + */ + @Test @Ignore + public void nodes_with_different_tags_and_one_tag_set() { + startNode(1, + GceComputeServiceTwoNodesDifferentTagsMock.class, + ImmutableSettings.settingsBuilder().put("discovery.gce.tags", "elasticsearch").build()); + startNode(2, + GceComputeServiceTwoNodesDifferentTagsMock.class, + ImmutableSettings.settingsBuilder().put("discovery.gce.tags", "elasticsearch").build()); + + // We expect having 1 nodes as part of the cluster, let's test that + checkNumberOfNodes(1); + } + + /** + * We need to ignore this test from elasticsearch version 1.2.1 as + * expected nodes running is 2 and this test will create 2 clusters with one node each. + * @see org.elasticsearch.test.ElasticsearchIntegrationTest#ensureClusterSizeConsistency() + * TODO Reactivate when it will be possible to set the number of running nodes + */ + @Test @Ignore + public void nodes_with_different_tags_and_two_tag_set() { + startNode(1, + GceComputeServiceTwoNodesDifferentTagsMock.class, + ImmutableSettings.settingsBuilder().put("discovery.gce.tags", Lists.newArrayList("elasticsearch", "dev")).build()); + startNode(2, + GceComputeServiceTwoNodesDifferentTagsMock.class, + ImmutableSettings.settingsBuilder().put("discovery.gce.tags", Lists.newArrayList("elasticsearch", "dev")).build()); + + // We expect having 1 nodes as part of the cluster, let's test that + checkNumberOfNodes(1); + } + + @Test @Ignore + public void nodes_with_same_tags_and_no_tag_set() { + startNode(1, + GceComputeServiceTwoNodesSameTagsMock.class, + ImmutableSettings.EMPTY); + startNode(2, + GceComputeServiceTwoNodesSameTagsMock.class, + ImmutableSettings.EMPTY); + + // We expect having 2 nodes as part of the cluster, let's test that + checkNumberOfNodes(2); + } + + @Test @Ignore + public void nodes_with_same_tags_and_one_tag_set() { + startNode(1, + GceComputeServiceTwoNodesSameTagsMock.class, + ImmutableSettings.settingsBuilder().put("discovery.gce.tags", "elasticsearch").build()); + startNode(2, + GceComputeServiceTwoNodesSameTagsMock.class, + ImmutableSettings.settingsBuilder().put("discovery.gce.tags", "elasticsearch").build()); + + // We expect having 2 nodes as part of the cluster, let's test that + checkNumberOfNodes(2); + } + + @Test @Ignore + public void nodes_with_same_tags_and_two_tags_set() { + startNode(1, + GceComputeServiceTwoNodesSameTagsMock.class, + ImmutableSettings.settingsBuilder().put("discovery.gce.tags", Lists.newArrayList("elasticsearch", "dev")).build()); + startNode(2, + GceComputeServiceTwoNodesSameTagsMock.class, + ImmutableSettings.settingsBuilder().put("discovery.gce.tags", Lists.newArrayList("elasticsearch", "dev")).build()); + + // We expect having 2 nodes as part of the cluster, let's test that + checkNumberOfNodes(2); + } +} diff --git a/src/test/java/org/elasticsearch/discovery/gce/GceDifferentTagsNoTagTest.java b/src/test/java/org/elasticsearch/discovery/gce/GceDifferentTagsNoTagTest.java deleted file mode 100644 index 481adb8ebcc..00000000000 --- a/src/test/java/org/elasticsearch/discovery/gce/GceDifferentTagsNoTagTest.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.discovery.gce; - -import org.elasticsearch.common.settings.ImmutableSettings; -import org.elasticsearch.common.settings.Settings; - -public class GceDifferentTagsNoTagTest extends AbstractGceComputeServiceTest { - - @Override - protected Settings nodeSettings(int nodeOrdinal) { - ImmutableSettings.Builder settings = ImmutableSettings.settingsBuilder().put(super.nodeSettings(nodeOrdinal)); - settings.put("discovery.type", "gce"); - settings.put("cloud.gce.api.impl", GceComputeServiceTwoNodesDifferentTagsMock.class); - return settings.build(); - } - - /** - * Set the number of expected nodes in the current cluster - */ - @Override - protected int getExpectedNodes() { - return 2; - } -} diff --git a/src/test/java/org/elasticsearch/discovery/gce/GceDifferentTagsOneTagTest.java b/src/test/java/org/elasticsearch/discovery/gce/GceDifferentTagsOneTagTest.java deleted file mode 100644 index b53a9e30ebd..00000000000 --- a/src/test/java/org/elasticsearch/discovery/gce/GceDifferentTagsOneTagTest.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.discovery.gce; - -import org.elasticsearch.common.settings.ImmutableSettings; -import org.elasticsearch.common.settings.Settings; -import org.junit.Ignore; - -/** - * We need to ignore this test from elasticsearch version 1.2.1 as - * expected nodes running is 2 and this test will create 2 clusters with one node each. - * @see org.elasticsearch.test.ElasticsearchIntegrationTest#ensureClusterSizeConsistency() - * TODO Reactivate when it will be possible to set the number of running nodes - */ -@Ignore -public class GceDifferentTagsOneTagTest extends AbstractGceComputeServiceTest { - - @Override - protected Settings nodeSettings(int nodeOrdinal) { - ImmutableSettings.Builder settings = ImmutableSettings.settingsBuilder().put(super.nodeSettings(nodeOrdinal)); - settings.put("discovery.type", "gce"); - settings.put("cloud.gce.api.impl", GceComputeServiceTwoNodesDifferentTagsMock.class); - settings.put("discovery.gce.tags", "elasticsearch"); - return settings.build(); - } - - /** - * Set the number of expected nodes in the current cluster - */ - @Override - protected int getExpectedNodes() { - return 1; - } -} diff --git a/src/test/java/org/elasticsearch/discovery/gce/GceDifferentTagsTwoTagsTest.java b/src/test/java/org/elasticsearch/discovery/gce/GceDifferentTagsTwoTagsTest.java deleted file mode 100644 index cbdd9c5542b..00000000000 --- a/src/test/java/org/elasticsearch/discovery/gce/GceDifferentTagsTwoTagsTest.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.discovery.gce; - -import org.elasticsearch.common.settings.ImmutableSettings; -import org.elasticsearch.common.settings.Settings; -import org.junit.Ignore; - -/** - * We need to ignore this test from elasticsearch version 1.2.1 as - * expected nodes running is 2 and this test will create 2 clusters with one node each. - * @see org.elasticsearch.test.ElasticsearchIntegrationTest#ensureClusterSizeConsistency() - * TODO Reactivate when it will be possible to set the number of running nodes - */ -@Ignore -public class GceDifferentTagsTwoTagsTest extends AbstractGceComputeServiceTest { - - @Override - protected Settings nodeSettings(int nodeOrdinal) { - ImmutableSettings.Builder settings = ImmutableSettings.settingsBuilder().put(super.nodeSettings(nodeOrdinal)); - settings.put("discovery.type", "gce"); - settings.put("cloud.gce.api.impl", GceComputeServiceTwoNodesDifferentTagsMock.class); - settings.put("discovery.gce.tags", "elasticsearch,dev"); - return settings.build(); - } - - /** - * Set the number of expected nodes in the current cluster - */ - @Override - protected int getExpectedNodes() { - return 1; - } -} diff --git a/src/test/java/org/elasticsearch/discovery/gce/GceSameTagsNoTagTest.java b/src/test/java/org/elasticsearch/discovery/gce/GceSameTagsNoTagTest.java deleted file mode 100644 index 055816475c9..00000000000 --- a/src/test/java/org/elasticsearch/discovery/gce/GceSameTagsNoTagTest.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.discovery.gce; - -import org.elasticsearch.common.settings.ImmutableSettings; -import org.elasticsearch.common.settings.Settings; - -public class GceSameTagsNoTagTest extends AbstractGceComputeServiceTest { - - @Override - protected Settings nodeSettings(int nodeOrdinal) { - ImmutableSettings.Builder settings = ImmutableSettings.settingsBuilder().put(super.nodeSettings(nodeOrdinal)); - settings.put("discovery.type", "gce"); - settings.put("cloud.gce.api.impl", GceComputeServiceTwoNodesSameTagsMock.class); - return settings.build(); - } - - /** - * Set the number of expected nodes in the current cluster - */ - @Override - protected int getExpectedNodes() { - return 2; - } -} diff --git a/src/test/java/org/elasticsearch/discovery/gce/GceSameTagsOneTagTest.java b/src/test/java/org/elasticsearch/discovery/gce/GceSameTagsOneTagTest.java deleted file mode 100644 index 7dbd2339d11..00000000000 --- a/src/test/java/org/elasticsearch/discovery/gce/GceSameTagsOneTagTest.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.discovery.gce; - -import org.elasticsearch.common.settings.ImmutableSettings; -import org.elasticsearch.common.settings.Settings; - -public class GceSameTagsOneTagTest extends AbstractGceComputeServiceTest { - - @Override - protected Settings nodeSettings(int nodeOrdinal) { - ImmutableSettings.Builder settings = ImmutableSettings.settingsBuilder().put(super.nodeSettings(nodeOrdinal)); - settings.put("discovery.type", "gce"); - settings.put("cloud.gce.api.impl", GceComputeServiceTwoNodesSameTagsMock.class); - settings.put("discovery.gce.tags", "elasticsearch"); - return settings.build(); - } - - /** - * Set the number of expected nodes in the current cluster - */ - @Override - protected int getExpectedNodes() { - return 2; - } -} diff --git a/src/test/java/org/elasticsearch/discovery/gce/GceSameTagsTwoTagsTest.java b/src/test/java/org/elasticsearch/discovery/gce/GceSameTagsTwoTagsTest.java deleted file mode 100644 index f2417a83657..00000000000 --- a/src/test/java/org/elasticsearch/discovery/gce/GceSameTagsTwoTagsTest.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.discovery.gce; - -import org.elasticsearch.common.settings.ImmutableSettings; -import org.elasticsearch.common.settings.Settings; - -public class GceSameTagsTwoTagsTest extends AbstractGceComputeServiceTest { - - @Override - protected Settings nodeSettings(int nodeOrdinal) { - ImmutableSettings.Builder settings = ImmutableSettings.settingsBuilder().put(super.nodeSettings(nodeOrdinal)); - settings.put("discovery.type", "gce"); - settings.put("cloud.gce.api.impl", GceComputeServiceTwoNodesSameTagsMock.class); - settings.put("discovery.gce.tags", "elasticsearch,dev"); - return settings.build(); - } - - /** - * Set the number of expected nodes in the current cluster - */ - @Override - protected int getExpectedNodes() { - return 2; - } -} diff --git a/src/test/java/org/elasticsearch/discovery/gce/GceComputeServiceAbstractMock.java b/src/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceAbstractMock.java similarity index 94% rename from src/test/java/org/elasticsearch/discovery/gce/GceComputeServiceAbstractMock.java rename to src/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceAbstractMock.java index 1cbe9bcc55a..af423f28186 100644 --- a/src/test/java/org/elasticsearch/discovery/gce/GceComputeServiceAbstractMock.java +++ b/src/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceAbstractMock.java @@ -17,7 +17,7 @@ * under the License. */ -package org.elasticsearch.discovery.gce; +package org.elasticsearch.discovery.gce.mock; import com.google.api.services.compute.model.Instance; import com.google.api.services.compute.model.Metadata; @@ -27,6 +27,7 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.cloud.gce.GceComputeService; import org.elasticsearch.common.component.AbstractLifecycleComponent; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.discovery.gce.GceComputeEngineTest; import java.util.ArrayList; import java.util.Collection; @@ -72,7 +73,7 @@ public abstract class GceComputeServiceAbstractMock extends AbstractLifecycleCom // Add metadata es_port:930X where X is the instance number Metadata metadata = new Metadata(); - metadata.put("es_port", "" + AbstractGceComputeServiceTest.getPort(nodeNumber)); + metadata.put("es_port", "" + GceComputeEngineTest.getPort(nodeNumber)); instance.setMetadata(metadata); instances.add(instance); diff --git a/src/test/java/org/elasticsearch/discovery/gce/GceComputeServiceTwoNodesDifferentTagsMock.java b/src/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceTwoNodesDifferentTagsMock.java similarity index 96% rename from src/test/java/org/elasticsearch/discovery/gce/GceComputeServiceTwoNodesDifferentTagsMock.java rename to src/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceTwoNodesDifferentTagsMock.java index 57878deb4ea..d8ec7c23f4f 100644 --- a/src/test/java/org/elasticsearch/discovery/gce/GceComputeServiceTwoNodesDifferentTagsMock.java +++ b/src/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceTwoNodesDifferentTagsMock.java @@ -17,7 +17,7 @@ * under the License. */ -package org.elasticsearch.discovery.gce; +package org.elasticsearch.discovery.gce.mock; import org.elasticsearch.common.collect.Lists; import org.elasticsearch.common.inject.Inject; diff --git a/src/test/java/org/elasticsearch/discovery/gce/GceComputeServiceTwoNodesSameTagsMock.java b/src/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceTwoNodesSameTagsMock.java similarity index 96% rename from src/test/java/org/elasticsearch/discovery/gce/GceComputeServiceTwoNodesSameTagsMock.java rename to src/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceTwoNodesSameTagsMock.java index 7b5be3687c5..d7d0cb56a1b 100644 --- a/src/test/java/org/elasticsearch/discovery/gce/GceComputeServiceTwoNodesSameTagsMock.java +++ b/src/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceTwoNodesSameTagsMock.java @@ -17,7 +17,7 @@ * under the License. */ -package org.elasticsearch.discovery.gce; +package org.elasticsearch.discovery.gce.mock; import org.elasticsearch.common.collect.Lists; import org.elasticsearch.common.inject.Inject; diff --git a/src/test/java/org/elasticsearch/gce/itest/GceSimpleITest.java b/src/test/java/org/elasticsearch/gce/itest/GceSimpleITest.java index f02aecc543b..c0d1ecdd5b2 100644 --- a/src/test/java/org/elasticsearch/gce/itest/GceSimpleITest.java +++ b/src/test/java/org/elasticsearch/gce/itest/GceSimpleITest.java @@ -30,7 +30,7 @@ import org.junit.Test; /** * This test needs GCE to run and -Dtests.gce=true to be set - * and -Des.config=/path/to/elasticsearch.yml + * and -Dtests.config=/path/to/elasticsearch.yml * TODO: By now, it will only work from GCE platform as we don't support yet external auth. * See https://github.com/elasticsearch/elasticsearch-cloud-gce/issues/10 * @see org.elasticsearch.cloud.gce.AbstractGceTest diff --git a/src/test/resources/elasticsearch.yml b/src/test/resources/elasticsearch.yml deleted file mode 100644 index 7fde927a0b6..00000000000 --- a/src/test/resources/elasticsearch.yml +++ /dev/null @@ -1,27 +0,0 @@ -# Licensed to ElasticSearch under one or more contributor -# license agreements. See the NOTICE file distributed with this work for additional -# information regarding copyright ownership. ElasticSearch licenses this file to you -# under the Apache License, Version 2.0 (the "License"); you may not use this -# file except in compliance with the License. You may obtain a copy of the -# License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by -# applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the specific language -# governing permissions and limitations under the License. - -# GCE discovery allows to use GCE API in order to perform discovery. -# -# You have to install the cloud-gce plugin for enabling the GCE discovery. -# -# See -# for more information. -# -# See README in elasticsearch-cloud-gce repository for a step-by-step tutorial. -# cloud: -# gce: -# project_id: es-cloud -# zone: europe-west1-a -# discovery: -# type: gce -# gce: -# tags: you_can_add_tags_here From 8e8e76fa75e838da362008e9de3bb1a076b4597d Mon Sep 17 00:00:00 2001 From: "Simon (Big Chief) Morley" Date: Wed, 6 Aug 2014 22:48:54 +0200 Subject: [PATCH 59/96] Add multiple zones support Closes #27. --- README.md | 20 ++++++++ .../cloud/gce/GceComputeServiceImpl.java | 48 +++++++++++++----- .../discovery/gce/GceComputeEngineTest.java | 33 +++++++++++++ .../mock/GceComputeServiceAbstractMock.java | 24 +++++++-- ...mputeServiceTwoNodesDifferentTagsMock.java | 5 ++ .../GceComputeServiceTwoNodesOneZoneMock.java | 49 +++++++++++++++++++ ...GceComputeServiceTwoNodesSameTagsMock.java | 5 ++ ...GceComputeServiceTwoNodesTwoZonesMock.java | 49 +++++++++++++++++++ 8 files changed, 215 insertions(+), 18 deletions(-) create mode 100644 src/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceTwoNodesOneZoneMock.java create mode 100644 src/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceTwoNodesTwoZonesMock.java diff --git a/README.md b/README.md index aef88c1a391..2b1895210e5 100644 --- a/README.md +++ b/README.md @@ -242,6 +242,26 @@ gcutil deletedisk boot-myesnode1 boot-myesnode2 \ --zone=europe-west1-a ``` +Using zones +----------- + +`cloud.gce.zone` helps to retrieve instances running in a given zone. It should be one of the +[GCE supported zones](https://developers.google.com/compute/docs/zones#available). + +The GCE discovery can support multi zones although you need to be aware of network latency between zones. +To enable discovery across more than one zone, just enter add your zone list to `cloud.gce.zone` setting: + +```yaml + cloud: + gce: + project_id: + zone: , + discovery: + type: gce +``` + + + Filtering by tags ----------------- diff --git a/src/main/java/org/elasticsearch/cloud/gce/GceComputeServiceImpl.java b/src/main/java/org/elasticsearch/cloud/gce/GceComputeServiceImpl.java index 5a08c5fefab..d4d6263a7ee 100644 --- a/src/main/java/org/elasticsearch/cloud/gce/GceComputeServiceImpl.java +++ b/src/main/java/org/elasticsearch/cloud/gce/GceComputeServiceImpl.java @@ -28,6 +28,9 @@ import com.google.api.services.compute.Compute; import com.google.api.services.compute.model.Instance; import com.google.api.services.compute.model.InstanceList; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.common.base.Function; +import org.elasticsearch.common.collect.Iterables; +import org.elasticsearch.common.collect.Lists; import org.elasticsearch.common.component.AbstractLifecycleComponent; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; @@ -36,8 +39,8 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.discovery.DiscoveryException; import java.io.IOException; -import java.util.ArrayList; import java.util.Collection; +import java.util.List; /** * @@ -46,7 +49,7 @@ public class GceComputeServiceImpl extends AbstractLifecycleComponent zoneList; // Forcing Google Token API URL as set in GCE SDK to // http://metadata/computeMetadata/v1/instance/service-accounts/default/token @@ -55,18 +58,34 @@ public class GceComputeServiceImpl extends AbstractLifecycleComponent instances() { - try { - logger.debug("get instances for project [{}], zone [{}]", project, zone); - Compute.Instances.List list = client().instances().list(project, zone); - InstanceList instanceList = list.execute(); + logger.debug("get instances for project [{}], zoneList [{}]", project, zoneList); - return instanceList.getItems(); - } catch (IOException e) { - logger.warn("disabling GCE discovery. Can not get list of nodes: {}", e.getMessage()); - logger.debug("Full exception:", e); - return new ArrayList(); - } + List> instanceListByZone = Lists.transform(zoneList, new Function>() { + @Override + public List apply(String zoneId) { + try { + Compute.Instances.List list = client().instances().list(project, zoneId); + InstanceList instanceList = list.execute(); + + return instanceList.getItems(); + } catch (IOException e) { + logger.warn("Problem fetching instance list for zone {}", zoneId); + logger.debug("Full exception:", e); + + return Lists.newArrayList(); + } + } + }); + + //Collapse instances from all zones into one neat list + List instanceList = Lists.newArrayList(Iterables.concat(instanceListByZone)); + + if (instanceList.size() == 0) { + logger.warn("disabling GCE discovery. Can not get list of nodes"); + } + + return instanceList; } private Compute client; @@ -85,7 +104,10 @@ public class GceComputeServiceImpl extends AbstractLifecycleComponent> getTags(); + protected abstract List getZones(); + private final List zoneList; protected GceComputeServiceAbstractMock(Settings settings) { super(settings); - logger.debug("starting GCE Api Mock with {} nodes:", getTags().size()); - for (List tags : getTags()) { - logger.debug(" - {}", tags); + + int numNodes = getTags().size() > getZones().size() ? getTags().size() : getZones().size(); + + logger.debug("starting GCE Api Mock with {} nodes:", numNodes); + for (int i = 0; i < numNodes; i++) { + List tags = getTags().size() > i ? getTags().get(i) : null; + String zone = getZones().size() > i ? getZones().get(i) : null; + logger.debug(" - node #{}: tags [{}], zone [{}]", i, tags, zone); } + + String[] zoneList = componentSettings.getAsArray(Fields.ZONE, settings.getAsArray("cloud.gce." + Fields.ZONE)); + this.zoneList = Lists.newArrayList(zoneList); } private Collection instances = null; private void computeInstances() { instances = new ArrayList(); - int nodeNumber = 0; // For each instance (item of tags) for (List tags : getTags()) { - logger.info(" ----> GCE Mock API: Adding node {}", nodeNumber); + String zone = zoneList.isEmpty() ? "dummy" : zoneList.get(randomInt(zoneList.size()-1)); + logger.info(" ----> GCE Mock API: Adding node [{}] in zone [{}]", nodeNumber, zone); Instance instance = new Instance(); instance.setName("Mock Node " + tags); instance.setMachineType("Mock Type machine"); instance.setStatus("STARTED"); + instance.setZone(zone); Tags instanceTags = new Tags(); instanceTags.setItems(tags); instance.setTags(instanceTags); diff --git a/src/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceTwoNodesDifferentTagsMock.java b/src/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceTwoNodesDifferentTagsMock.java index d8ec7c23f4f..c261e2ea3d8 100644 --- a/src/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceTwoNodesDifferentTagsMock.java +++ b/src/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceTwoNodesDifferentTagsMock.java @@ -39,6 +39,11 @@ public class GceComputeServiceTwoNodesDifferentTagsMock extends GceComputeServic return tags; } + @Override + protected List getZones() { + return Lists.newArrayList(); + } + @Inject protected GceComputeServiceTwoNodesDifferentTagsMock(Settings settings) { super(settings); diff --git a/src/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceTwoNodesOneZoneMock.java b/src/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceTwoNodesOneZoneMock.java new file mode 100644 index 00000000000..5a87c6fcbf6 --- /dev/null +++ b/src/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceTwoNodesOneZoneMock.java @@ -0,0 +1,49 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.discovery.gce.mock; + +import org.elasticsearch.common.collect.Lists; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; + +import java.util.ArrayList; +import java.util.List; + +/** + * + */ +public class GceComputeServiceTwoNodesOneZoneMock extends GceComputeServiceAbstractMock { + private static List zones = Lists.newArrayList("us-central1-a","us-central1-a"); + + @Override + protected List> getTags() { + return Lists.newArrayList(); + } + + @Override + protected List getZones() { + return zones; + } + + @Inject + protected GceComputeServiceTwoNodesOneZoneMock(Settings settings) { + super(settings); + } +} diff --git a/src/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceTwoNodesSameTagsMock.java b/src/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceTwoNodesSameTagsMock.java index d7d0cb56a1b..4de658f284a 100644 --- a/src/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceTwoNodesSameTagsMock.java +++ b/src/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceTwoNodesSameTagsMock.java @@ -40,6 +40,11 @@ public class GceComputeServiceTwoNodesSameTagsMock extends GceComputeServiceAbst return tags; } + @Override + protected List getZones() { + return Lists.newArrayList(); + } + @Inject protected GceComputeServiceTwoNodesSameTagsMock(Settings settings) { super(settings); diff --git a/src/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceTwoNodesTwoZonesMock.java b/src/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceTwoNodesTwoZonesMock.java new file mode 100644 index 00000000000..527e508d844 --- /dev/null +++ b/src/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceTwoNodesTwoZonesMock.java @@ -0,0 +1,49 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.discovery.gce.mock; + +import org.elasticsearch.common.collect.Lists; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; + +import java.util.ArrayList; +import java.util.List; + +/** + * + */ +public class GceComputeServiceTwoNodesTwoZonesMock extends GceComputeServiceAbstractMock { + private static List zones = Lists.newArrayList("us-central1-a","europe-west1-a"); + + @Override + protected List> getTags() { + return Lists.newArrayList(); + } + + @Override + protected List getZones() { + return zones; + } + + @Inject + protected GceComputeServiceTwoNodesTwoZonesMock(Settings settings) { + super(settings); + } +} From 7018d04b7844219f9f2ff68dc19e42ee50eaca2d Mon Sep 17 00:00:00 2001 From: "Simon (Big Chief) Morley" Date: Wed, 6 Aug 2014 23:07:24 +0200 Subject: [PATCH 60/96] Update to elasticsearch 1.4.0 Closes #34. (cherry picked from commit e149f70) --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 2b1895210e5..5ceac266db9 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,7 @@ You need to install a version matching your Elasticsearch version: | Elasticsearch | GCE Cloud Plugin | Docs | |------------------------|-------------------|------------------------------------------------------------------------------------------------------------------------------------| | master | Build from source | See below | +| es-1.x | Build from source | [2.4.0-SNAPSHOT](https://github.com/elasticsearch/elasticsearch-cloud-gce/tree/es-1.x/#google-compute-engine-cloud-plugin-for-elasticsearch)| | es-1.3 | Build from source | [2.3.0-SNAPSHOT](https://github.com/elasticsearch/elasticsearch-cloud-gce/tree/es-1.3/#google-compute-engine-cloud-plugin-for-elasticsearch)| | es-1.2 | 2.2.0 | [2.2.0](https://github.com/elasticsearch/elasticsearch-cloud-gce/tree/v2.2.0/#google-compute-engine-cloud-plugin-for-elasticsearch)| | es-1.1 | 2.1.2 | [2.1.2](https://github.com/elasticsearch/elasticsearch-cloud-gce/tree/v2.1.2/#google-compute-engine-cloud-plugin-for-elasticsearch)| From 7b4fa1f0a0b59c7fdcb1264306ab3d58d354c71d Mon Sep 17 00:00:00 2001 From: David Pilato Date: Wed, 6 Aug 2014 23:14:47 +0200 Subject: [PATCH 61/96] update documentation with release 2.3.0 --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 5ceac266db9..76a5e7a6dfc 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ The GCE Cloud plugin allows to use GCE API for the unicast discovery mechanism. In order to install the plugin, run: ```sh -bin/plugin -install elasticsearch/elasticsearch-cloud-gce/2.2.0 +bin/plugin -install elasticsearch/elasticsearch-cloud-gce/2.3.0 ``` You need to install a version matching your Elasticsearch version: @@ -15,7 +15,7 @@ You need to install a version matching your Elasticsearch version: |------------------------|-------------------|------------------------------------------------------------------------------------------------------------------------------------| | master | Build from source | See below | | es-1.x | Build from source | [2.4.0-SNAPSHOT](https://github.com/elasticsearch/elasticsearch-cloud-gce/tree/es-1.x/#google-compute-engine-cloud-plugin-for-elasticsearch)| -| es-1.3 | Build from source | [2.3.0-SNAPSHOT](https://github.com/elasticsearch/elasticsearch-cloud-gce/tree/es-1.3/#google-compute-engine-cloud-plugin-for-elasticsearch)| +| es-1.3 | 2.3.0 | [2.3.0](https://github.com/elasticsearch/elasticsearch-cloud-gce/tree/v2.3.0/#version-230-for-elasticsearch-13) | | es-1.2 | 2.2.0 | [2.2.0](https://github.com/elasticsearch/elasticsearch-cloud-gce/tree/v2.2.0/#google-compute-engine-cloud-plugin-for-elasticsearch)| | es-1.1 | 2.1.2 | [2.1.2](https://github.com/elasticsearch/elasticsearch-cloud-gce/tree/v2.1.2/#google-compute-engine-cloud-plugin-for-elasticsearch)| | es-1.0 | 2.0.1 | [2.0.1](https://github.com/elasticsearch/elasticsearch-cloud-gce/tree/v2.0.1/#google-compute-engine-cloud-plugin-for-elasticsearch)| From d951483c4c8d653994bc80871af32782e0d0f1fc Mon Sep 17 00:00:00 2001 From: David Pilato Date: Fri, 5 Sep 2014 16:51:31 +0200 Subject: [PATCH 62/96] ZenDiscovery constructor needs ElectMasterService instance Introduced in https://github.com/elasticsearch/elasticsearch/pull/7336 (elasticsearch 1.4 and 2.0), we need to change GceDiscovery constructor. Closes #35. --- .../java/org/elasticsearch/discovery/gce/GceDiscovery.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/elasticsearch/discovery/gce/GceDiscovery.java b/src/main/java/org/elasticsearch/discovery/gce/GceDiscovery.java index 3c4ab62896d..31b75ad062b 100755 --- a/src/main/java/org/elasticsearch/discovery/gce/GceDiscovery.java +++ b/src/main/java/org/elasticsearch/discovery/gce/GceDiscovery.java @@ -30,6 +30,7 @@ import org.elasticsearch.common.network.NetworkService; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.discovery.DiscoverySettings; import org.elasticsearch.discovery.zen.ZenDiscovery; +import org.elasticsearch.discovery.zen.elect.ElectMasterService; import org.elasticsearch.discovery.zen.ping.ZenPing; import org.elasticsearch.discovery.zen.ping.ZenPingService; import org.elasticsearch.discovery.zen.ping.unicast.UnicastZenPing; @@ -46,9 +47,10 @@ public class GceDiscovery extends ZenDiscovery { public GceDiscovery(Settings settings, ClusterName clusterName, ThreadPool threadPool, TransportService transportService, ClusterService clusterService, NodeSettingsService nodeSettingsService, ZenPingService pingService, DiscoveryNodeService discoveryNodeService, GceComputeService gceComputeService, - NetworkService networkService, DiscoverySettings discoverySettings) { + NetworkService networkService, DiscoverySettings discoverySettings, + ElectMasterService electMasterService) { super(settings, clusterName, threadPool, transportService, clusterService, nodeSettingsService, - discoveryNodeService, pingService, Version.CURRENT, discoverySettings); + discoveryNodeService, pingService, electMasterService, Version.CURRENT, discoverySettings); if (settings.getAsBoolean("cloud.enabled", true)) { ImmutableList zenPings = pingService.zenPings(); UnicastZenPing unicastZenPing = null; From 09f83018489dfbeabe059ad7c78999e6bd9e8fe6 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Mon, 15 Sep 2014 14:35:37 +0200 Subject: [PATCH 63/96] Create branch es-1.4 for elasticsearch 1.4.0 --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 76a5e7a6dfc..3e3286414f5 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,8 @@ You need to install a version matching your Elasticsearch version: | Elasticsearch | GCE Cloud Plugin | Docs | |------------------------|-------------------|------------------------------------------------------------------------------------------------------------------------------------| | master | Build from source | See below | -| es-1.x | Build from source | [2.4.0-SNAPSHOT](https://github.com/elasticsearch/elasticsearch-cloud-gce/tree/es-1.x/#google-compute-engine-cloud-plugin-for-elasticsearch)| +| es-1.x | Build from source | [2.5.0-SNAPSHOT](https://github.com/elasticsearch/elasticsearch-cloud-gce/tree/es-1.x/#google-compute-engine-cloud-plugin-for-elasticsearch)| +| es-1.4 | Build from source | [2.4.0-SNAPSHOT](https://github.com/elasticsearch/elasticsearch-cloud-gce/tree/es-1.4/#google-compute-engine-cloud-plugin-for-elasticsearch)| | es-1.3 | 2.3.0 | [2.3.0](https://github.com/elasticsearch/elasticsearch-cloud-gce/tree/v2.3.0/#version-230-for-elasticsearch-13) | | es-1.2 | 2.2.0 | [2.2.0](https://github.com/elasticsearch/elasticsearch-cloud-gce/tree/v2.2.0/#google-compute-engine-cloud-plugin-for-elasticsearch)| | es-1.1 | 2.1.2 | [2.1.2](https://github.com/elasticsearch/elasticsearch-cloud-gce/tree/v2.1.2/#google-compute-engine-cloud-plugin-for-elasticsearch)| From 00eeb866cf01aa246363f55529ab2ddec08a8e47 Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Wed, 5 Nov 2014 17:56:00 -0500 Subject: [PATCH 64/96] upgrade to lucene 5 snapshot (the compile is still broken as it was before, so untested...) --- pom.xml | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index c8d1b8f1085..3e52ea840c4 100644 --- a/pom.xml +++ b/pom.xml @@ -51,7 +51,8 @@ governing permissions and limitations under the License. --> 2.0.0-SNAPSHOT v1-rev15-1.18.0-rc 1.18.0-rc - 4.9.0 + 5.0.0 + 5.0.0-snapshot-1636426 true onerror @@ -60,6 +61,17 @@ governing permissions and limitations under the License. --> ${project.build.testOutputDirectory}/elasticsearch.yml + + + sonatype + http://oss.sonatype.org/content/repositories/releases/ + + + Lucene snapshots + https://download.elasticsearch.org/lucenesnapshots/maven/ + + + org.hamcrest @@ -77,7 +89,7 @@ governing permissions and limitations under the License. --> org.apache.lucene lucene-test-framework - ${lucene.version} + ${lucene.maven.version} test From 4e520c10ea4458ab64bc04c7b06c9c2e0b13d1d5 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Wed, 5 Nov 2014 15:29:23 -0800 Subject: [PATCH 65/96] Fix constructor for ZenDiscovery's subclass --- .../java/org/elasticsearch/discovery/gce/GceDiscovery.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/elasticsearch/discovery/gce/GceDiscovery.java b/src/main/java/org/elasticsearch/discovery/gce/GceDiscovery.java index 31b75ad062b..3f09735cc97 100755 --- a/src/main/java/org/elasticsearch/discovery/gce/GceDiscovery.java +++ b/src/main/java/org/elasticsearch/discovery/gce/GceDiscovery.java @@ -19,11 +19,11 @@ package org.elasticsearch.discovery.gce; -import org.elasticsearch.Version; import org.elasticsearch.cloud.gce.GceComputeService; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.node.DiscoveryNodeService; +import org.elasticsearch.cluster.settings.DynamicSettings; import org.elasticsearch.common.collect.ImmutableList; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.network.NetworkService; @@ -48,9 +48,9 @@ public class GceDiscovery extends ZenDiscovery { ClusterService clusterService, NodeSettingsService nodeSettingsService, ZenPingService pingService, DiscoveryNodeService discoveryNodeService, GceComputeService gceComputeService, NetworkService networkService, DiscoverySettings discoverySettings, - ElectMasterService electMasterService) { + ElectMasterService electMasterService, DynamicSettings dynamicSettings) { super(settings, clusterName, threadPool, transportService, clusterService, nodeSettingsService, - discoveryNodeService, pingService, electMasterService, Version.CURRENT, discoverySettings); + discoveryNodeService, pingService, electMasterService, discoverySettings, dynamicSettings); if (settings.getAsBoolean("cloud.enabled", true)) { ImmutableList zenPings = pingService.zenPings(); UnicastZenPing unicastZenPing = null; From 0ef9028134172d7757fefa907e794e4585e7263b Mon Sep 17 00:00:00 2001 From: jdyer09 Date: Fri, 24 Oct 2014 11:39:19 -0600 Subject: [PATCH 66/96] Update README.md for zone clarity --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 3e3286414f5..cb2239ed0f5 100644 --- a/README.md +++ b/README.md @@ -257,7 +257,7 @@ To enable discovery across more than one zone, just enter add your zone list to cloud: gce: project_id: - zone: , + zone: ["", ""] discovery: type: gce ``` From d79ae5d39a864bc3f1cb6e26377993f829c10544 Mon Sep 17 00:00:00 2001 From: Michael McCandless Date: Mon, 10 Nov 2014 16:46:31 -0500 Subject: [PATCH 67/96] Upgrade to Lucene 5.0.0-snapshot-1637347 --- pom.xml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pom.xml b/pom.xml index 3e52ea840c4..b69ed661528 100644 --- a/pom.xml +++ b/pom.xml @@ -52,7 +52,7 @@ governing permissions and limitations under the License. --> v1-rev15-1.18.0-rc 1.18.0-rc 5.0.0 - 5.0.0-snapshot-1636426 + 5.0.0-snapshot-1637347 true onerror @@ -63,12 +63,12 @@ governing permissions and limitations under the License. --> - sonatype - http://oss.sonatype.org/content/repositories/releases/ + Lucene snapshots + https://download.elasticsearch.org/lucenesnapshots/1637347/ - Lucene snapshots - https://download.elasticsearch.org/lucenesnapshots/maven/ + sonatype + http://oss.sonatype.org/content/repositories/releases/ From de0458ed935c07431278758d6e52885e9029f6f3 Mon Sep 17 00:00:00 2001 From: tlrx Date: Wed, 12 Nov 2014 14:13:53 +0100 Subject: [PATCH 68/96] update documentation with release 2.4.0 --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index cb2239ed0f5..259e4cf7644 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ The GCE Cloud plugin allows to use GCE API for the unicast discovery mechanism. In order to install the plugin, run: ```sh -bin/plugin -install elasticsearch/elasticsearch-cloud-gce/2.3.0 +bin/plugin -install elasticsearch/elasticsearch-cloud-gce/2.4.0 ``` You need to install a version matching your Elasticsearch version: @@ -15,7 +15,7 @@ You need to install a version matching your Elasticsearch version: |------------------------|-------------------|------------------------------------------------------------------------------------------------------------------------------------| | master | Build from source | See below | | es-1.x | Build from source | [2.5.0-SNAPSHOT](https://github.com/elasticsearch/elasticsearch-cloud-gce/tree/es-1.x/#google-compute-engine-cloud-plugin-for-elasticsearch)| -| es-1.4 | Build from source | [2.4.0-SNAPSHOT](https://github.com/elasticsearch/elasticsearch-cloud-gce/tree/es-1.4/#google-compute-engine-cloud-plugin-for-elasticsearch)| +| es-1.4 | 2.4.0 | [2.4.0](https://github.com/elasticsearch/elasticsearch-cloud-gce/tree/v2.4.0/#version-240-for-elasticsearch-14) | | es-1.3 | 2.3.0 | [2.3.0](https://github.com/elasticsearch/elasticsearch-cloud-gce/tree/v2.3.0/#version-230-for-elasticsearch-13) | | es-1.2 | 2.2.0 | [2.2.0](https://github.com/elasticsearch/elasticsearch-cloud-gce/tree/v2.2.0/#google-compute-engine-cloud-plugin-for-elasticsearch)| | es-1.1 | 2.1.2 | [2.1.2](https://github.com/elasticsearch/elasticsearch-cloud-gce/tree/v2.1.2/#google-compute-engine-cloud-plugin-for-elasticsearch)| From 554d942acfb8eef93bad37bceb225cba00137ff3 Mon Sep 17 00:00:00 2001 From: tlrx Date: Wed, 12 Nov 2014 14:08:50 +0100 Subject: [PATCH 69/96] [TESTS] Upgrade randomizedtesting-runner to 2.1.10 (cherry picked from commit 87b1b25) --- pom.xml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pom.xml b/pom.xml index b69ed661528..35f40fa2bf9 100644 --- a/pom.xml +++ b/pom.xml @@ -138,6 +138,13 @@ governing permissions and limitations under the License. --> test + + com.carrotsearch.randomizedtesting + randomizedtesting-runner + 2.1.10 + test + + From 9529d735be49a98856d8ffa1ab8bcb65f0355d32 Mon Sep 17 00:00:00 2001 From: Michael McCandless Date: Mon, 24 Nov 2014 05:52:16 -0500 Subject: [PATCH 70/96] Upgrade to Lucene 5.0.0-snapshot-1641343 --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 35f40fa2bf9..1fd85a7644c 100644 --- a/pom.xml +++ b/pom.xml @@ -52,7 +52,7 @@ governing permissions and limitations under the License. --> v1-rev15-1.18.0-rc 1.18.0-rc 5.0.0 - 5.0.0-snapshot-1637347 + 5.0.0-snapshot-1641343 true onerror @@ -64,7 +64,7 @@ governing permissions and limitations under the License. --> Lucene snapshots - https://download.elasticsearch.org/lucenesnapshots/1637347/ + https://download.elasticsearch.org/lucenesnapshots/1641343/ sonatype From e2343d694d93261996f42a87e41dd4a2d676369c Mon Sep 17 00:00:00 2001 From: David Pilato Date: Mon, 24 Nov 2014 18:28:05 +0100 Subject: [PATCH 71/96] UnicastHostsProvider should use version.minimumCompatibilityVersion() The UnicastHostsProvider implementation creates DiscoveryNodes that are used as an initial seed for unicast based discovery. At the moment it uses Version.CURRENT for those DiscoveryNode object, which confuses the backwards compatibility layer to think this nodes are of the latest version. This causes new nodes to fail to join old nodes as the ping serialization goes wrong. Instead we should use version.minimumCompatibilityVersion(). Closes #41. (cherry picked from commit 9ae29c7) --- .../java/org/elasticsearch/discovery/gce/GceDiscovery.java | 6 ++++-- .../discovery/gce/GceUnicastHostsProvider.java | 7 +++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/elasticsearch/discovery/gce/GceDiscovery.java b/src/main/java/org/elasticsearch/discovery/gce/GceDiscovery.java index 3f09735cc97..a2b4ae1c44b 100755 --- a/src/main/java/org/elasticsearch/discovery/gce/GceDiscovery.java +++ b/src/main/java/org/elasticsearch/discovery/gce/GceDiscovery.java @@ -19,6 +19,7 @@ package org.elasticsearch.discovery.gce; +import org.elasticsearch.Version; import org.elasticsearch.cloud.gce.GceComputeService; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterService; @@ -48,7 +49,8 @@ public class GceDiscovery extends ZenDiscovery { ClusterService clusterService, NodeSettingsService nodeSettingsService, ZenPingService pingService, DiscoveryNodeService discoveryNodeService, GceComputeService gceComputeService, NetworkService networkService, DiscoverySettings discoverySettings, - ElectMasterService electMasterService, DynamicSettings dynamicSettings) { + ElectMasterService electMasterService, DynamicSettings dynamicSettings, + Version version) { super(settings, clusterName, threadPool, transportService, clusterService, nodeSettingsService, discoveryNodeService, pingService, electMasterService, discoverySettings, dynamicSettings); if (settings.getAsBoolean("cloud.enabled", true)) { @@ -65,7 +67,7 @@ public class GceDiscovery extends ZenDiscovery { // update the unicast zen ping to add cloud hosts provider // and, while we are at it, use only it and not the multicast for example unicastZenPing.addHostsProvider(new GceUnicastHostsProvider(settings, gceComputeService, - transportService, networkService)); + transportService, networkService, version)); pingService.zenPings(ImmutableList.of(unicastZenPing)); } else { logger.warn("failed to apply gce unicast discovery, no unicast ping found"); diff --git a/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java b/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java index d2508d4ffd4..3f2ddc51e74 100644 --- a/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java +++ b/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java @@ -56,6 +56,7 @@ public class GceUnicastHostsProvider extends AbstractComponent implements Unicas private TransportService transportService; private NetworkService networkService; + private final Version version; private final String project; private final String zone; private final String[] tags; @@ -67,11 +68,13 @@ public class GceUnicastHostsProvider extends AbstractComponent implements Unicas @Inject public GceUnicastHostsProvider(Settings settings, GceComputeService gceComputeService, TransportService transportService, - NetworkService networkService) { + NetworkService networkService, + Version version) { super(settings); this.gceComputeService = gceComputeService; this.transportService = transportService; this.networkService = networkService; + this.version = version; this.refreshInterval = componentSettings.getAsTime(Fields.REFRESH, settings.getAsTime("cloud.gce." + Fields.REFRESH, TimeValue.timeValueSeconds(0))); @@ -228,7 +231,7 @@ public class GceUnicastHostsProvider extends AbstractComponent implements Unicas // we only limit to 1 addresses, makes no sense to ping 100 ports logger.trace("adding {}, type {}, address {}, transport_address {}, status {}", name, type, ip_private, addresses[0], status); - cachedDiscoNodes.add(new DiscoveryNode("#cloud-" + name + "-" + 0, addresses[0], Version.CURRENT)); + cachedDiscoNodes.add(new DiscoveryNode("#cloud-" + name + "-" + 0, addresses[0], version.minimumCompatibilityVersion())); } } catch (Exception e) { logger.warn("failed to add {}, address {}", e, name, ip_private); From b26f92dc0f9e974384ade42f681389a481a0516d Mon Sep 17 00:00:00 2001 From: David Pilato Date: Mon, 24 Nov 2014 18:38:56 +0100 Subject: [PATCH 72/96] update documentation with release 2.4.1 --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 259e4cf7644..7239e88595e 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ The GCE Cloud plugin allows to use GCE API for the unicast discovery mechanism. In order to install the plugin, run: ```sh -bin/plugin -install elasticsearch/elasticsearch-cloud-gce/2.4.0 +bin/plugin install elasticsearch/elasticsearch-cloud-gce/2.4.1 ``` You need to install a version matching your Elasticsearch version: @@ -15,7 +15,7 @@ You need to install a version matching your Elasticsearch version: |------------------------|-------------------|------------------------------------------------------------------------------------------------------------------------------------| | master | Build from source | See below | | es-1.x | Build from source | [2.5.0-SNAPSHOT](https://github.com/elasticsearch/elasticsearch-cloud-gce/tree/es-1.x/#google-compute-engine-cloud-plugin-for-elasticsearch)| -| es-1.4 | 2.4.0 | [2.4.0](https://github.com/elasticsearch/elasticsearch-cloud-gce/tree/v2.4.0/#version-240-for-elasticsearch-14) | +| es-1.4 | 2.4.1 | [2.4.1](https://github.com/elasticsearch/elasticsearch-cloud-gce/tree/v2.4.1/#version-241-for-elasticsearch-14) | | es-1.3 | 2.3.0 | [2.3.0](https://github.com/elasticsearch/elasticsearch-cloud-gce/tree/v2.3.0/#version-230-for-elasticsearch-13) | | es-1.2 | 2.2.0 | [2.2.0](https://github.com/elasticsearch/elasticsearch-cloud-gce/tree/v2.2.0/#google-compute-engine-cloud-plugin-for-elasticsearch)| | es-1.1 | 2.1.2 | [2.1.2](https://github.com/elasticsearch/elasticsearch-cloud-gce/tree/v2.1.2/#google-compute-engine-cloud-plugin-for-elasticsearch)| From ced093fec212c954c3af717b70612cc5bf8dd501 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Tue, 2 Dec 2014 18:22:17 +0100 Subject: [PATCH 73/96] Upgrade to Lucene 5.0.0-snapshot-1642891 --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 1fd85a7644c..5e869ed7dcb 100644 --- a/pom.xml +++ b/pom.xml @@ -52,7 +52,7 @@ governing permissions and limitations under the License. --> v1-rev15-1.18.0-rc 1.18.0-rc 5.0.0 - 5.0.0-snapshot-1641343 + 5.0.0-snapshot-1642891 true onerror @@ -64,7 +64,7 @@ governing permissions and limitations under the License. --> Lucene snapshots - https://download.elasticsearch.org/lucenesnapshots/1641343/ + https://download.elasticsearch.org/lucenesnapshots/1642891/ sonatype From 7839cdf3c23930e67b18cbddc4a49bc1ca5d3939 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Fri, 2 Jan 2015 21:30:46 +0100 Subject: [PATCH 74/96] Depend on elasticsearch-parent To simplify plugins maintenance and provide more value in the future, we are starting to build an `elasticsearch-parent` project. This commit is the first step for this plugin to depend on this new `pom` maven project. --- .gitignore | 8 ++-- dev-tools/tests.policy | 54 +++++++++++++++++++++++++ pom.xml | 91 ++++++++++-------------------------------- 3 files changed, 78 insertions(+), 75 deletions(-) create mode 100644 dev-tools/tests.policy diff --git a/.gitignore b/.gitignore index 38d7701b94d..9533848e238 100644 --- a/.gitignore +++ b/.gitignore @@ -5,9 +5,9 @@ /target .DS_Store *.iml -/.settings /.project +/.settings /.classpath -*.vmoptions -.local-execution-hints.log -plugin_tools +/plugin_tools +/.local-execution-hints.log +/.local-*-execution-hints.log diff --git a/dev-tools/tests.policy b/dev-tools/tests.policy new file mode 100644 index 00000000000..6afb5025840 --- /dev/null +++ b/dev-tools/tests.policy @@ -0,0 +1,54 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// Policy file to prevent tests from writing outside the test sandbox directory +// PLEASE NOTE: You may need to enable other permissions when new tests are added, +// everything not allowed here is forbidden! + +grant { + // permissions for file access, write access only to sandbox: + permission java.io.FilePermission "<>", "read,execute"; + permission java.io.FilePermission "${junit4.childvm.cwd}", "read,execute,write"; + permission java.io.FilePermission "${junit4.childvm.cwd}${/}-", "read,execute,write,delete"; + permission java.io.FilePermission "${junit4.tempDir}${/}*", "read,execute,write,delete"; + permission groovy.security.GroovyCodeSourcePermission "/groovy/script"; + + // Allow connecting to the internet anywhere + permission java.net.SocketPermission "*", "accept,listen,connect,resolve"; + + // Basic permissions needed for Lucene / Elasticsearch to work: + permission java.util.PropertyPermission "*", "read,write"; + permission java.lang.reflect.ReflectPermission "*"; + permission java.lang.RuntimePermission "*"; + + // These two *have* to be spelled out a separate + permission java.lang.management.ManagementPermission "control"; + permission java.lang.management.ManagementPermission "monitor"; + + permission java.net.NetPermission "*"; + permission java.util.logging.LoggingPermission "control"; + permission javax.management.MBeanPermission "*", "*"; + permission javax.management.MBeanServerPermission "*"; + permission javax.management.MBeanTrustPermission "*"; + + // Needed for some things in DNS caching in the JVM + permission java.security.SecurityPermission "getProperty.networkaddress.cache.ttl"; + permission java.security.SecurityPermission "getProperty.networkaddress.cache.negative.ttl"; + +}; diff --git a/pom.xml b/pom.xml index 5e869ed7dcb..f81b6040745 100644 --- a/pom.xml +++ b/pom.xml @@ -14,6 +14,13 @@ governing permissions and limitations under the License. --> xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 + + + org.elasticsearch + elasticsearch-parent + 2.0.0-SNAPSHOT + + org.elasticsearch elasticsearch-cloud-gce 3.0.0-SNAPSHOT @@ -23,12 +30,6 @@ governing permissions and limitations under the License. --> https://github.com/elasticsearch/elasticsearch-cloud-gce/ 2013 - - org.sonatype.oss - oss-parent - 7 - - The Apache Software License, Version 2.0 @@ -48,56 +49,28 @@ governing permissions and limitations under the License. --> - 2.0.0-SNAPSHOT v1-rev15-1.18.0-rc 1.18.0-rc - 5.0.0 - 5.0.0-snapshot-1642891 - true - onerror - 9300 - INFO - ${project.build.testOutputDirectory}/elasticsearch.yml - - - Lucene snapshots - https://download.elasticsearch.org/lucenesnapshots/1642891/ - - - sonatype - http://oss.sonatype.org/content/repositories/releases/ - - - org.hamcrest - hamcrest-core - 1.3.RC2 - test + hamcrest-all - org.hamcrest - hamcrest-library - 1.3.RC2 - test + com.carrotsearch.randomizedtesting + randomizedtesting-runner - org.apache.lucene lucene-test-framework - ${lucene.maven.version} - test - - org.elasticsearch elasticsearch - ${elasticsearch.version} + provided @@ -126,25 +99,14 @@ governing permissions and limitations under the License. --> log4j log4j - 1.2.17 test org.elasticsearch elasticsearch - ${elasticsearch.version} test-jar - test - - - com.carrotsearch.randomizedtesting - randomizedtesting-runner - 2.1.10 - test - - @@ -183,16 +145,10 @@ governing permissions and limitations under the License. --> org.apache.maven.plugins maven-compiler-plugin - 3.1 - - 1.6 - 1.6 - com.carrotsearch.randomizedtesting junit4-maven-plugin - 2.0.14 tests @@ -270,8 +226,6 @@ governing permissions and limitations under the License. --> ${tests.integration} ${tests.cluster_seed} ${tests.client.ratio} - false - network ${es.logger.level} true @@ -282,27 +236,14 @@ governing permissions and limitations under the License. --> org.apache.maven.plugins maven-surefire-plugin - 2.15 - - true - org.apache.maven.plugins maven-source-plugin - 2.2.1 - - - attach-sources - - jar - - - + org.apache.maven.plugins maven-assembly-plugin - 2.4 false ${project.build.directory}/releases/ @@ -321,4 +262,12 @@ governing permissions and limitations under the License. --> + + + + oss-snapshots + Sonatype OSS Snapshots + https://oss.sonatype.org/content/repositories/snapshots/ + + From 3fd35d2694db3176209e54c015689e5998aab50c Mon Sep 17 00:00:00 2001 From: David Pilato Date: Mon, 16 Mar 2015 16:34:27 -0700 Subject: [PATCH 75/96] create `es-1.5` branch --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 7239e88595e..bb237f948d3 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,8 @@ You need to install a version matching your Elasticsearch version: | Elasticsearch | GCE Cloud Plugin | Docs | |------------------------|-------------------|------------------------------------------------------------------------------------------------------------------------------------| | master | Build from source | See below | -| es-1.x | Build from source | [2.5.0-SNAPSHOT](https://github.com/elasticsearch/elasticsearch-cloud-gce/tree/es-1.x/#google-compute-engine-cloud-plugin-for-elasticsearch)| +| es-1.x | Build from source | [2.6.0-SNAPSHOT](https://github.com/elasticsearch/elasticsearch-cloud-gce/tree/es-1.x/#google-compute-engine-cloud-plugin-for-elasticsearch)| +| es-1.5 | Build from source | [2.5.0-SNAPSHOT](https://github.com/elasticsearch/elasticsearch-cloud-gce/tree/es-1.5/#google-compute-engine-cloud-plugin-for-elasticsearch)| | es-1.4 | 2.4.1 | [2.4.1](https://github.com/elasticsearch/elasticsearch-cloud-gce/tree/v2.4.1/#version-241-for-elasticsearch-14) | | es-1.3 | 2.3.0 | [2.3.0](https://github.com/elasticsearch/elasticsearch-cloud-gce/tree/v2.3.0/#version-230-for-elasticsearch-13) | | es-1.2 | 2.2.0 | [2.2.0](https://github.com/elasticsearch/elasticsearch-cloud-gce/tree/v2.2.0/#google-compute-engine-cloud-plugin-for-elasticsearch)| From 49b7eefc8dfec7bd304ff53ef3c9f4445463874d Mon Sep 17 00:00:00 2001 From: David Pilato Date: Wed, 25 Mar 2015 15:36:19 +0100 Subject: [PATCH 76/96] Discovery NPE when a zone has 0 nodes in it Closes #43. (cherry picked from commit b028910) (cherry picked from commit fa3e157) --- .../cloud/gce/GceComputeServiceImpl.java | 3 + .../discovery/gce/GceComputeEngineTest.java | 24 ++++-- .../mock/GceComputeServiceZeroNodeMock.java | 78 +++++++++++++++++++ 3 files changed, 97 insertions(+), 8 deletions(-) create mode 100644 src/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceZeroNodeMock.java diff --git a/src/main/java/org/elasticsearch/cloud/gce/GceComputeServiceImpl.java b/src/main/java/org/elasticsearch/cloud/gce/GceComputeServiceImpl.java index d4d6263a7ee..2d8ca9fcae9 100644 --- a/src/main/java/org/elasticsearch/cloud/gce/GceComputeServiceImpl.java +++ b/src/main/java/org/elasticsearch/cloud/gce/GceComputeServiceImpl.java @@ -67,6 +67,9 @@ public class GceComputeServiceImpl extends AbstractLifecycleComponent> getTags() { + return Lists.newArrayList(); + } + + @Override + protected List getZones() { + return Lists.newArrayList(); + } + + private final List zoneList; + + @Override + public Collection instances() { + logger.debug("get instances for zoneList [{}]", zoneList); + + List> instanceListByZone = Lists.transform(zoneList, new Function>() { + @Override + public List apply(String zoneId) { + // If we return null here we will get a trace as explained in issue 43 + return Lists.newArrayList(); + } + }); + + //Collapse instances from all zones into one neat list + List instanceList = Lists.newArrayList(Iterables.concat(instanceListByZone)); + + if (instanceList.size() == 0) { + logger.warn("disabling GCE discovery. Can not get list of nodes"); + } + + return instanceList; + } + + @Inject + protected GceComputeServiceZeroNodeMock(Settings settings) { + super(settings); + String[] zoneList = componentSettings.getAsArray(Fields.ZONE, settings.getAsArray("cloud.gce." + Fields.ZONE)); + this.zoneList = Lists.newArrayList(zoneList); + } +} From d6231e2c9fdf609c36bf8e9978c5e49699153fb6 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Wed, 25 Mar 2015 15:40:58 +0100 Subject: [PATCH 77/96] Move parent after artifact coordinates --- pom.xml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pom.xml b/pom.xml index f81b6040745..c5c278d64be 100644 --- a/pom.xml +++ b/pom.xml @@ -15,12 +15,6 @@ governing permissions and limitations under the License. --> xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - - org.elasticsearch - elasticsearch-parent - 2.0.0-SNAPSHOT - - org.elasticsearch elasticsearch-cloud-gce 3.0.0-SNAPSHOT @@ -43,6 +37,12 @@ governing permissions and limitations under the License. --> http://github.com/elasticsearch/elasticsearch-cloud-gce + + org.elasticsearch + elasticsearch-parent + 2.0.0-SNAPSHOT + + GitHub https://github.com/elasticsearch/elasticsearch-cloud-gce/issues/ From 2c94de30e959d6f0c864bfd89eb21913e87aebf7 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Wed, 25 Mar 2015 14:52:04 +0100 Subject: [PATCH 78/96] Move to elastic owner (cherry picked from commit d745edb) (cherry picked from commit 8231854) --- pom.xml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pom.xml b/pom.xml index c5c278d64be..9bd27bf94ed 100644 --- a/pom.xml +++ b/pom.xml @@ -21,7 +21,7 @@ governing permissions and limitations under the License. --> Elasticsearch Google Compute Engine cloud plugin The Google Compute Engine (GCE) Cloud plugin allows to use GCE API for the unicast discovery mechanism. - https://github.com/elasticsearch/elasticsearch-cloud-gce/ + https://github.com/elastic/elasticsearch-cloud-gce/ 2013 @@ -32,9 +32,9 @@ governing permissions and limitations under the License. --> - scm:git:git@github.com:elasticsearch/elasticsearch-cloud-gce.git - scm:git:git@github.com:elasticsearch/elasticsearch-cloud-gce.git - http://github.com/elasticsearch/elasticsearch-cloud-gce + scm:git:git@github.com:elastic/elasticsearch-cloud-gce.git + scm:git:git@github.com:elastic/elasticsearch-cloud-gce.git + http://github.com/elastic/elasticsearch-cloud-gce @@ -45,7 +45,7 @@ governing permissions and limitations under the License. --> GitHub - https://github.com/elasticsearch/elasticsearch-cloud-gce/issues/ + https://github.com/elastic/elasticsearch-cloud-gce/issues/ From 75a8d1272ed3c35b392f63fd8ca6891bd3390245 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Wed, 25 Mar 2015 15:48:59 +0100 Subject: [PATCH 79/96] update documentation with release 2.5.0 --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index bb237f948d3..8f6dd29e855 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ The GCE Cloud plugin allows to use GCE API for the unicast discovery mechanism. In order to install the plugin, run: ```sh -bin/plugin install elasticsearch/elasticsearch-cloud-gce/2.4.1 +bin/plugin install elasticsearch/elasticsearch-cloud-gce/2.5.0 ``` You need to install a version matching your Elasticsearch version: @@ -15,7 +15,7 @@ You need to install a version matching your Elasticsearch version: |------------------------|-------------------|------------------------------------------------------------------------------------------------------------------------------------| | master | Build from source | See below | | es-1.x | Build from source | [2.6.0-SNAPSHOT](https://github.com/elasticsearch/elasticsearch-cloud-gce/tree/es-1.x/#google-compute-engine-cloud-plugin-for-elasticsearch)| -| es-1.5 | Build from source | [2.5.0-SNAPSHOT](https://github.com/elasticsearch/elasticsearch-cloud-gce/tree/es-1.5/#google-compute-engine-cloud-plugin-for-elasticsearch)| +| es-1.5 | 2.5.0 | [2.5.0](https://github.com/elastic/elasticsearch-cloud-gce/tree/v2.5.0/#version-250-for-elasticsearch-15) | | es-1.4 | 2.4.1 | [2.4.1](https://github.com/elasticsearch/elasticsearch-cloud-gce/tree/v2.4.1/#version-241-for-elasticsearch-14) | | es-1.3 | 2.3.0 | [2.3.0](https://github.com/elasticsearch/elasticsearch-cloud-gce/tree/v2.3.0/#version-230-for-elasticsearch-13) | | es-1.2 | 2.2.0 | [2.2.0](https://github.com/elasticsearch/elasticsearch-cloud-gce/tree/v2.2.0/#google-compute-engine-cloud-plugin-for-elasticsearch)| From ebe75a33e40670c939044d59ea63e3a6f6d128b6 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Tue, 21 Apr 2015 21:33:56 +0200 Subject: [PATCH 80/96] Fix non working update dynamic settings Described in https://github.com/elastic/elasticsearch/issues/10614, it's not possible with cloud discovery plugin to update dynamic settings anymore. ```sh curl -XPUT localhost:9200/_cluster/settings -d '{ "persistent" : { "discovery.zen.minimum_master_nodes" : 3 }, "transient" : { "discovery.zen.minimum_master_nodes" : 3 } }' ``` gives ```json {"acknowledged":true,"persistent":{},"transient":{}} ``` Closes #47. (cherry picked from commit 9df33a3) (cherry picked from commit a40016e) --- .../java/org/elasticsearch/discovery/gce/GceDiscovery.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/elasticsearch/discovery/gce/GceDiscovery.java b/src/main/java/org/elasticsearch/discovery/gce/GceDiscovery.java index a2b4ae1c44b..f8ca2861130 100755 --- a/src/main/java/org/elasticsearch/discovery/gce/GceDiscovery.java +++ b/src/main/java/org/elasticsearch/discovery/gce/GceDiscovery.java @@ -24,6 +24,7 @@ import org.elasticsearch.cloud.gce.GceComputeService; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.node.DiscoveryNodeService; +import org.elasticsearch.cluster.settings.ClusterDynamicSettings; import org.elasticsearch.cluster.settings.DynamicSettings; import org.elasticsearch.common.collect.ImmutableList; import org.elasticsearch.common.inject.Inject; @@ -49,7 +50,7 @@ public class GceDiscovery extends ZenDiscovery { ClusterService clusterService, NodeSettingsService nodeSettingsService, ZenPingService pingService, DiscoveryNodeService discoveryNodeService, GceComputeService gceComputeService, NetworkService networkService, DiscoverySettings discoverySettings, - ElectMasterService electMasterService, DynamicSettings dynamicSettings, + ElectMasterService electMasterService, @ClusterDynamicSettings DynamicSettings dynamicSettings, Version version) { super(settings, clusterName, threadPool, transportService, clusterService, nodeSettingsService, discoveryNodeService, pingService, electMasterService, discoverySettings, dynamicSettings); From 7efb77b24f06c9c3441682eb1be4f62ed723fe5f Mon Sep 17 00:00:00 2001 From: David Pilato Date: Fri, 24 Apr 2015 12:38:49 +0200 Subject: [PATCH 81/96] Update to GCE API v1-rev59-1.20.0 GCE API have been updated to `v1-rev59-1.20.0`. We now can depend only on: ```xml com.google.apis google-api-services-compute v1-rev59-1.20.0 ``` It comes with all needed dependencies. Closes #49. (cherry picked from commit f95dc16) (cherry picked from commit 4ee6c95) --- pom.xml | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/pom.xml b/pom.xml index 9bd27bf94ed..775f396bc1c 100644 --- a/pom.xml +++ b/pom.xml @@ -49,8 +49,7 @@ governing permissions and limitations under the License. --> - v1-rev15-1.18.0-rc - 1.18.0-rc + v1-rev59-1.20.0 9300 @@ -74,21 +73,6 @@ governing permissions and limitations under the License. --> - - com.google.api-client - google-api-client - ${google.api.version} - - - com.google.http-client - google-http-client-jackson2 - ${google.api.version} - - - com.google.oauth-client - google-oauth-client-jetty - ${google.api.version} - com.google.apis google-api-services-compute From 947569a56678d6e8d0538cd19f3646003aff8222 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Fri, 24 Apr 2015 12:39:53 +0200 Subject: [PATCH 82/96] Remove oss snapshot repo as defined in parent (cherry picked from commit 20e154e) (cherry picked from commit 994c8ef) --- pom.xml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/pom.xml b/pom.xml index 775f396bc1c..f1679a5823e 100644 --- a/pom.xml +++ b/pom.xml @@ -247,11 +247,4 @@ governing permissions and limitations under the License. --> - - - oss-snapshots - Sonatype OSS Snapshots - https://oss.sonatype.org/content/repositories/snapshots/ - - From c70e32d1198815e84bc785851eb1a41d6c179c3b Mon Sep 17 00:00:00 2001 From: David Pilato Date: Fri, 24 Apr 2015 14:37:19 +0200 Subject: [PATCH 83/96] Remove component settings from AbstractComponent Related to elasticsearch/elasticsearch#9919 Closes #50. --- .../cloud/gce/GceComputeService.java | 7 ++-- .../cloud/gce/GceComputeServiceImpl.java | 12 +++---- .../cloud/gce/GceSettingsFilter.java | 35 ------------------ .../discovery/gce/GceDiscovery.java | 36 +++---------------- .../gce/GceUnicastHostsProvider.java | 12 +++---- .../discovery/gce/GceComputeEngineTest.java | 27 +++++++------- .../mock/GceComputeServiceAbstractMock.java | 2 +- .../mock/GceComputeServiceZeroNodeMock.java | 2 +- 8 files changed, 34 insertions(+), 99 deletions(-) delete mode 100644 src/main/java/org/elasticsearch/cloud/gce/GceSettingsFilter.java diff --git a/src/main/java/org/elasticsearch/cloud/gce/GceComputeService.java b/src/main/java/org/elasticsearch/cloud/gce/GceComputeService.java index cb0e35654a7..b6b45b2861a 100644 --- a/src/main/java/org/elasticsearch/cloud/gce/GceComputeService.java +++ b/src/main/java/org/elasticsearch/cloud/gce/GceComputeService.java @@ -28,9 +28,10 @@ import java.util.Collection; */ public interface GceComputeService { static final public class Fields { - public static final String PROJECT = "project_id"; - public static final String ZONE = "zone"; - public static final String REFRESH = "refresh_interval"; + public static final String PROJECT = "cloud.gce.project_id"; + public static final String ZONE = "cloud.gce.zone"; + public static final String REFRESH = "cloud.gce.refresh_interval"; + public static final String TAGS = "discovery.gce.tags"; public static final String VERSION = "Elasticsearch/GceCloud/1.0"; } diff --git a/src/main/java/org/elasticsearch/cloud/gce/GceComputeServiceImpl.java b/src/main/java/org/elasticsearch/cloud/gce/GceComputeServiceImpl.java index 2d8ca9fcae9..00ac256a79a 100644 --- a/src/main/java/org/elasticsearch/cloud/gce/GceComputeServiceImpl.java +++ b/src/main/java/org/elasticsearch/cloud/gce/GceComputeServiceImpl.java @@ -28,6 +28,7 @@ import com.google.api.services.compute.Compute; import com.google.api.services.compute.model.Instance; import com.google.api.services.compute.model.InstanceList; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.ElasticsearchIllegalArgumentException; import org.elasticsearch.common.base.Function; import org.elasticsearch.common.collect.Iterables; import org.elasticsearch.common.collect.Lists; @@ -36,7 +37,6 @@ import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.discovery.DiscoveryException; import java.io.IOException; import java.util.Collection; @@ -104,13 +104,9 @@ public class GceComputeServiceImpl extends AbstractLifecycleComponent zenPings = pingService.zenPings(); - UnicastZenPing unicastZenPing = null; - for (ZenPing zenPing : zenPings) { - if (zenPing instanceof UnicastZenPing) { - unicastZenPing = (UnicastZenPing) zenPing; - break; - } - } + pingService, electMasterService, discoverySettings, dynamicSettings); - if (unicastZenPing != null) { - // update the unicast zen ping to add cloud hosts provider - // and, while we are at it, use only it and not the multicast for example - unicastZenPing.addHostsProvider(new GceUnicastHostsProvider(settings, gceComputeService, - transportService, networkService, version)); - pingService.zenPings(ImmutableList.of(unicastZenPing)); - } else { - logger.warn("failed to apply gce unicast discovery, no unicast ping found"); - } - } + // TODO Add again force disable multicast + // See related issue in AWS plugin https://github.com/elastic/elasticsearch-cloud-aws/issues/179 } } diff --git a/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java b/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java index 3f2ddc51e74..1326279ac72 100644 --- a/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java +++ b/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java @@ -76,17 +76,15 @@ public class GceUnicastHostsProvider extends AbstractComponent implements Unicas this.networkService = networkService; this.version = version; - this.refreshInterval = componentSettings.getAsTime(Fields.REFRESH, - settings.getAsTime("cloud.gce." + Fields.REFRESH, TimeValue.timeValueSeconds(0))); - - this.project = componentSettings.get(Fields.PROJECT, settings.get("cloud.gce." + Fields.PROJECT)); - this.zone = componentSettings.get(Fields.ZONE, settings.get("cloud.gce." + Fields.ZONE)); + this.refreshInterval = settings.getAsTime(Fields.REFRESH, TimeValue.timeValueSeconds(0)); + this.project = settings.get(Fields.PROJECT); + this.zone = settings.get(Fields.ZONE); // Check that we have all needed properties checkProperty(Fields.PROJECT, project); checkProperty(Fields.ZONE, zone); - this.tags = settings.getAsArray("discovery.gce.tags"); + this.tags = settings.getAsArray(Fields.TAGS); if (logger.isDebugEnabled()) { logger.debug("using tags {}", Lists.newArrayList(this.tags)); } @@ -251,7 +249,7 @@ public class GceUnicastHostsProvider extends AbstractComponent implements Unicas private void checkProperty(String name, String value) { if (!Strings.hasText(value)) { - logger.warn("cloud.gce.{} is not set.", name); + logger.warn("{} is not set.", name); } } } diff --git a/src/test/java/org/elasticsearch/discovery/gce/GceComputeEngineTest.java b/src/test/java/org/elasticsearch/discovery/gce/GceComputeEngineTest.java index 72489e38283..35467c25c98 100644 --- a/src/test/java/org/elasticsearch/discovery/gce/GceComputeEngineTest.java +++ b/src/test/java/org/elasticsearch/discovery/gce/GceComputeEngineTest.java @@ -21,6 +21,7 @@ package org.elasticsearch.discovery.gce; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; import org.elasticsearch.cloud.gce.GceComputeService; +import org.elasticsearch.cloud.gce.GceComputeService.Fields; import org.elasticsearch.common.collect.Lists; import org.elasticsearch.common.settings.ImmutableSettings; import org.elasticsearch.common.settings.Settings; @@ -112,10 +113,10 @@ public class GceComputeEngineTest extends ElasticsearchIntegrationTest { public void nodes_with_different_tags_and_one_tag_set() { startNode(1, GceComputeServiceTwoNodesDifferentTagsMock.class, - ImmutableSettings.settingsBuilder().put("discovery.gce.tags", "elasticsearch").build()); + ImmutableSettings.settingsBuilder().put(Fields.TAGS, "elasticsearch").build()); startNode(2, GceComputeServiceTwoNodesDifferentTagsMock.class, - ImmutableSettings.settingsBuilder().put("discovery.gce.tags", "elasticsearch").build()); + ImmutableSettings.settingsBuilder().put(Fields.TAGS, "elasticsearch").build()); // We expect having 1 nodes as part of the cluster, let's test that checkNumberOfNodes(1); @@ -131,10 +132,10 @@ public class GceComputeEngineTest extends ElasticsearchIntegrationTest { public void nodes_with_different_tags_and_two_tag_set() { startNode(1, GceComputeServiceTwoNodesDifferentTagsMock.class, - ImmutableSettings.settingsBuilder().put("discovery.gce.tags", Lists.newArrayList("elasticsearch", "dev")).build()); + ImmutableSettings.settingsBuilder().put(Fields.TAGS, Lists.newArrayList("elasticsearch", "dev")).build()); startNode(2, GceComputeServiceTwoNodesDifferentTagsMock.class, - ImmutableSettings.settingsBuilder().put("discovery.gce.tags", Lists.newArrayList("elasticsearch", "dev")).build()); + ImmutableSettings.settingsBuilder().put(Fields.TAGS, Lists.newArrayList("elasticsearch", "dev")).build()); // We expect having 1 nodes as part of the cluster, let's test that checkNumberOfNodes(1); @@ -157,10 +158,10 @@ public class GceComputeEngineTest extends ElasticsearchIntegrationTest { public void nodes_with_same_tags_and_one_tag_set() { startNode(1, GceComputeServiceTwoNodesSameTagsMock.class, - ImmutableSettings.settingsBuilder().put("discovery.gce.tags", "elasticsearch").build()); + ImmutableSettings.settingsBuilder().put(Fields.TAGS, "elasticsearch").build()); startNode(2, GceComputeServiceTwoNodesSameTagsMock.class, - ImmutableSettings.settingsBuilder().put("discovery.gce.tags", "elasticsearch").build()); + ImmutableSettings.settingsBuilder().put(Fields.TAGS, "elasticsearch").build()); // We expect having 2 nodes as part of the cluster, let's test that checkNumberOfNodes(2); @@ -170,10 +171,10 @@ public class GceComputeEngineTest extends ElasticsearchIntegrationTest { public void nodes_with_same_tags_and_two_tags_set() { startNode(1, GceComputeServiceTwoNodesSameTagsMock.class, - ImmutableSettings.settingsBuilder().put("discovery.gce.tags", Lists.newArrayList("elasticsearch", "dev")).build()); + ImmutableSettings.settingsBuilder().put(Fields.TAGS, Lists.newArrayList("elasticsearch", "dev")).build()); startNode(2, GceComputeServiceTwoNodesSameTagsMock.class, - ImmutableSettings.settingsBuilder().put("discovery.gce.tags", Lists.newArrayList("elasticsearch", "dev")).build()); + ImmutableSettings.settingsBuilder().put(Fields.TAGS, Lists.newArrayList("elasticsearch", "dev")).build()); // We expect having 2 nodes as part of the cluster, let's test that checkNumberOfNodes(2); @@ -183,11 +184,11 @@ public class GceComputeEngineTest extends ElasticsearchIntegrationTest { public void multiple_zones_and_two_nodes_in_same_zone() { startNode(1, GceComputeServiceTwoNodesOneZoneMock.class, - ImmutableSettings.settingsBuilder().put("cloud.gce.zone", Lists.newArrayList("us-central1-a", "us-central1-b", + ImmutableSettings.settingsBuilder().put(Fields.ZONE, Lists.newArrayList("us-central1-a", "us-central1-b", "us-central1-f", "europe-west1-a", "europe-west1-b")).build()); startNode(2, GceComputeServiceTwoNodesOneZoneMock.class, - ImmutableSettings.settingsBuilder().put("cloud.gce.zone", Lists.newArrayList("us-central1-a", "us-central1-b", + ImmutableSettings.settingsBuilder().put(Fields.ZONE, Lists.newArrayList("us-central1-a", "us-central1-b", "us-central1-f", "europe-west1-a", "europe-west1-b")).build()); // We expect having 2 nodes as part of the cluster, let's test that @@ -198,11 +199,11 @@ public class GceComputeEngineTest extends ElasticsearchIntegrationTest { public void multiple_zones_and_two_nodes_in_different_zones() { startNode(1, GceComputeServiceTwoNodesTwoZonesMock.class, - ImmutableSettings.settingsBuilder().put("cloud.gce.zone", Lists.newArrayList("us-central1-a", "us-central1-b", + ImmutableSettings.settingsBuilder().put(Fields.ZONE, Lists.newArrayList("us-central1-a", "us-central1-b", "us-central1-f", "europe-west1-a", "europe-west1-b")).build()); startNode(2, GceComputeServiceTwoNodesTwoZonesMock.class, - ImmutableSettings.settingsBuilder().put("cloud.gce.zone", Lists.newArrayList("us-central1-a", "us-central1-b", + ImmutableSettings.settingsBuilder().put(Fields.ZONE, Lists.newArrayList("us-central1-a", "us-central1-b", "us-central1-f", "europe-west1-a", "europe-west1-b")).build()); // We expect having 2 nodes as part of the cluster, let's test that @@ -216,7 +217,7 @@ public class GceComputeEngineTest extends ElasticsearchIntegrationTest { public void zero_node_43() { startNode(1, GceComputeServiceZeroNodeMock.class, - ImmutableSettings.settingsBuilder().put("cloud.gce.zone", Lists.newArrayList("us-central1-a", "us-central1-b", + ImmutableSettings.settingsBuilder().put(Fields.ZONE, Lists.newArrayList("us-central1-a", "us-central1-b", "us-central1-f", "europe-west1-a", "europe-west1-b")).build()); } diff --git a/src/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceAbstractMock.java b/src/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceAbstractMock.java index 4f7dfc121fe..7b5a19461b1 100644 --- a/src/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceAbstractMock.java +++ b/src/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceAbstractMock.java @@ -58,7 +58,7 @@ public abstract class GceComputeServiceAbstractMock extends AbstractLifecycleCom logger.debug(" - node #{}: tags [{}], zone [{}]", i, tags, zone); } - String[] zoneList = componentSettings.getAsArray(Fields.ZONE, settings.getAsArray("cloud.gce." + Fields.ZONE)); + String[] zoneList = settings.getAsArray(Fields.ZONE); this.zoneList = Lists.newArrayList(zoneList); } diff --git a/src/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceZeroNodeMock.java b/src/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceZeroNodeMock.java index 5a92861a3c2..6fd22a47c0b 100644 --- a/src/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceZeroNodeMock.java +++ b/src/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceZeroNodeMock.java @@ -72,7 +72,7 @@ public class GceComputeServiceZeroNodeMock extends GceComputeServiceAbstractMock @Inject protected GceComputeServiceZeroNodeMock(Settings settings) { super(settings); - String[] zoneList = componentSettings.getAsArray(Fields.ZONE, settings.getAsArray("cloud.gce." + Fields.ZONE)); + String[] zoneList = settings.getAsArray(Fields.ZONE); this.zoneList = Lists.newArrayList(zoneList); } } From bee93a51fed2d18bb883d99658ed0f3a0dc7e651 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Wed, 29 Apr 2015 12:26:44 +0200 Subject: [PATCH 84/96] Remove `ElasticsearchIllegalArgumentException` and `ElasticsearchIllegalStateException` in favor of the JDK one Related to https://github.com/elastic/elasticsearch/issues/10794 Closes #51. --- .../org/elasticsearch/cloud/gce/GceComputeServiceImpl.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/java/org/elasticsearch/cloud/gce/GceComputeServiceImpl.java b/src/main/java/org/elasticsearch/cloud/gce/GceComputeServiceImpl.java index 00ac256a79a..a786041c31f 100644 --- a/src/main/java/org/elasticsearch/cloud/gce/GceComputeServiceImpl.java +++ b/src/main/java/org/elasticsearch/cloud/gce/GceComputeServiceImpl.java @@ -28,7 +28,6 @@ import com.google.api.services.compute.Compute; import com.google.api.services.compute.model.Instance; import com.google.api.services.compute.model.InstanceList; import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.ElasticsearchIllegalArgumentException; import org.elasticsearch.common.base.Function; import org.elasticsearch.common.collect.Iterables; import org.elasticsearch.common.collect.Lists; @@ -141,7 +140,7 @@ public class GceComputeServiceImpl extends AbstractLifecycleComponent Date: Tue, 5 May 2015 14:22:53 -0400 Subject: [PATCH 85/96] Tests: fix compilation --- src/test/java/org/elasticsearch/cloud/gce/AbstractGceTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/test/java/org/elasticsearch/cloud/gce/AbstractGceTest.java b/src/test/java/org/elasticsearch/cloud/gce/AbstractGceTest.java index 6388a2c03f9..80070e47a62 100644 --- a/src/test/java/org/elasticsearch/cloud/gce/AbstractGceTest.java +++ b/src/test/java/org/elasticsearch/cloud/gce/AbstractGceTest.java @@ -57,9 +57,10 @@ public abstract class AbstractGceTest extends ElasticsearchIntegrationTest { protected Settings nodeSettings(int nodeOrdinal) { ImmutableSettings.Builder settings = ImmutableSettings.builder() .put(super.nodeSettings(nodeOrdinal)) + .put("path.home", createTempDir()) .put("plugins." + PluginsService.LOAD_PLUGIN_FROM_CLASSPATH, true); - Environment environment = new Environment(); + Environment environment = new Environment(settings.build()); // if explicit, just load it and don't load from env try { From 53bf03fbca77963e38a54487acaa9923c907e65f Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Mon, 11 May 2015 19:48:54 -0400 Subject: [PATCH 86/96] enable security manager in tests --- dev-tools/tests.policy | 54 ------------------------------------------ pom.xml | 1 + 2 files changed, 1 insertion(+), 54 deletions(-) delete mode 100644 dev-tools/tests.policy diff --git a/dev-tools/tests.policy b/dev-tools/tests.policy deleted file mode 100644 index 6afb5025840..00000000000 --- a/dev-tools/tests.policy +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -// Policy file to prevent tests from writing outside the test sandbox directory -// PLEASE NOTE: You may need to enable other permissions when new tests are added, -// everything not allowed here is forbidden! - -grant { - // permissions for file access, write access only to sandbox: - permission java.io.FilePermission "<>", "read,execute"; - permission java.io.FilePermission "${junit4.childvm.cwd}", "read,execute,write"; - permission java.io.FilePermission "${junit4.childvm.cwd}${/}-", "read,execute,write,delete"; - permission java.io.FilePermission "${junit4.tempDir}${/}*", "read,execute,write,delete"; - permission groovy.security.GroovyCodeSourcePermission "/groovy/script"; - - // Allow connecting to the internet anywhere - permission java.net.SocketPermission "*", "accept,listen,connect,resolve"; - - // Basic permissions needed for Lucene / Elasticsearch to work: - permission java.util.PropertyPermission "*", "read,write"; - permission java.lang.reflect.ReflectPermission "*"; - permission java.lang.RuntimePermission "*"; - - // These two *have* to be spelled out a separate - permission java.lang.management.ManagementPermission "control"; - permission java.lang.management.ManagementPermission "monitor"; - - permission java.net.NetPermission "*"; - permission java.util.logging.LoggingPermission "control"; - permission javax.management.MBeanPermission "*", "*"; - permission javax.management.MBeanServerPermission "*"; - permission javax.management.MBeanTrustPermission "*"; - - // Needed for some things in DNS caching in the JVM - permission java.security.SecurityPermission "getProperty.networkaddress.cache.ttl"; - permission java.security.SecurityPermission "getProperty.networkaddress.cache.negative.ttl"; - -}; diff --git a/pom.xml b/pom.xml index f1679a5823e..d341186fe99 100644 --- a/pom.xml +++ b/pom.xml @@ -51,6 +51,7 @@ governing permissions and limitations under the License. --> v1-rev59-1.20.0 9300 + true From 7f65196ee835c55eff7838b3feded7eec2131fc8 Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Mon, 11 May 2015 21:26:36 -0400 Subject: [PATCH 87/96] remove unnecessary prop --- pom.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/pom.xml b/pom.xml index d341186fe99..f1679a5823e 100644 --- a/pom.xml +++ b/pom.xml @@ -51,7 +51,6 @@ governing permissions and limitations under the License. --> v1-rev59-1.20.0 9300 - true From e2675c36165d13e16612389a14562a4fbec39b70 Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Wed, 13 May 2015 22:46:49 -0400 Subject: [PATCH 88/96] Use ThirdParty annotation and remove duplicate test config. See elastic/elasticsearch-parent#41 --- pom.xml | 85 +------------------ .../cloud/gce/AbstractGceTest.java | 27 +----- .../gce/itest/GceSimpleITest.java | 1 - 3 files changed, 6 insertions(+), 107 deletions(-) diff --git a/pom.xml b/pom.xml index f1679a5823e..0ceedee9006 100644 --- a/pom.xml +++ b/pom.xml @@ -51,6 +51,8 @@ governing permissions and limitations under the License. --> v1-rev59-1.20.0 9300 + + warn @@ -133,89 +135,6 @@ governing permissions and limitations under the License. --> com.carrotsearch.randomizedtesting junit4-maven-plugin - - - tests - test - - junit4 - - - 20 - pipe,warn - true - - - - - - - - - 1 - - - - - - - **/*Tests.class - **/*Test.class - - - **/Abstract*.class - **/*StressTest.class - - - ${tests.jvm.argline} - - - -Xmx512m - -Xss256k - -XX:MaxDirectMemorySize=512m - -Des.logger.prefix= - - ${tests.shuffle} - ${tests.verbose} - ${tests.seed} - ${tests.failfast} - - - ${tests.jvm.argline} - ${tests.iters} - ${tests.maxfailures} - ${tests.failfast} - ${tests.class} - ${tests.method} - ${tests.nightly} - ${tests.badapples} - ${tests.weekly} - ${tests.slow} - ${tests.gce} - ${tests.config} - ${tests.awaitsfix} - ${tests.slow} - ${tests.timeoutSuite} - ${tests.showSuccess} - ${tests.integration} - ${tests.cluster_seed} - ${tests.client.ratio} - ${es.logger.level} - true - - - - org.apache.maven.plugins diff --git a/src/test/java/org/elasticsearch/cloud/gce/AbstractGceTest.java b/src/test/java/org/elasticsearch/cloud/gce/AbstractGceTest.java index 80070e47a62..b17199e415d 100644 --- a/src/test/java/org/elasticsearch/cloud/gce/AbstractGceTest.java +++ b/src/test/java/org/elasticsearch/cloud/gce/AbstractGceTest.java @@ -19,7 +19,6 @@ package org.elasticsearch.cloud.gce; -import com.carrotsearch.randomizedtesting.annotations.TestGroup; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.ImmutableSettings; import org.elasticsearch.common.settings.Settings; @@ -27,32 +26,14 @@ import org.elasticsearch.env.Environment; import org.elasticsearch.env.FailedToResolveConfigException; import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.test.ElasticsearchIntegrationTest; - -import java.lang.annotation.Documented; -import java.lang.annotation.Inherited; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; +import org.elasticsearch.test.ElasticsearchIntegrationTest.ThirdParty; /** * */ +@ThirdParty public abstract class AbstractGceTest extends ElasticsearchIntegrationTest { - /** - * Annotation for tests that require GCE to run. GCE tests are disabled by default. - * See README file for details. - */ - @Documented - @Inherited - @Retention(RetentionPolicy.RUNTIME) - @TestGroup(enabled = false, sysProperty = SYSPROP_GCE) - public @interface GceTest { - } - - /** - */ - public static final String SYSPROP_GCE = "tests.gce"; - @Override protected Settings nodeSettings(int nodeOrdinal) { ImmutableSettings.Builder settings = ImmutableSettings.builder() @@ -67,10 +48,10 @@ public abstract class AbstractGceTest extends ElasticsearchIntegrationTest { if (Strings.hasText(System.getProperty("tests.config"))) { settings.loadFromUrl(environment.resolveConfig(System.getProperty("tests.config"))); } else { - fail("to run integration tests, you need to set -Dtest.gce=true and -Dtests.config=/path/to/elasticsearch.yml"); + throw new IllegalStateException("to run integration tests, you need to set -Dtests.thirdparty=true and -Dtests.config=/path/to/elasticsearch.yml"); } } catch (FailedToResolveConfigException exception) { - fail("your test configuration file is incorrect: " + System.getProperty("tests.config")); + throw new IllegalStateException("your test configuration file is incorrect: " + System.getProperty("tests.config"), exception); } return settings.build(); } diff --git a/src/test/java/org/elasticsearch/gce/itest/GceSimpleITest.java b/src/test/java/org/elasticsearch/gce/itest/GceSimpleITest.java index c0d1ecdd5b2..896f98a76c4 100644 --- a/src/test/java/org/elasticsearch/gce/itest/GceSimpleITest.java +++ b/src/test/java/org/elasticsearch/gce/itest/GceSimpleITest.java @@ -35,7 +35,6 @@ import org.junit.Test; * See https://github.com/elasticsearch/elasticsearch-cloud-gce/issues/10 * @see org.elasticsearch.cloud.gce.AbstractGceTest */ -@AbstractGceTest.GceTest @ElasticsearchIntegrationTest.ClusterScope( scope = ElasticsearchIntegrationTest.Scope.SUITE, numDataNodes = 1, From 5f423da9112b3ec48eda935946a03a8d253ff95b Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Wed, 13 May 2015 23:17:37 -0400 Subject: [PATCH 89/96] Use tests.jvms=1 like the old parallelism of 1 --- pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/pom.xml b/pom.xml index 0ceedee9006..cef2ece56c7 100644 --- a/pom.xml +++ b/pom.xml @@ -53,6 +53,7 @@ governing permissions and limitations under the License. --> 9300 warn + 1 From bc2f603ae3d8f584e977b87658739883b732ab93 Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Thu, 21 May 2015 10:34:18 -0400 Subject: [PATCH 90/96] engage forbidden apis --- pom.xml | 61 ++------------------------------------------------------- 1 file changed, 2 insertions(+), 59 deletions(-) diff --git a/pom.xml b/pom.xml index cef2ece56c7..3a3772c4cc4 100644 --- a/pom.xml +++ b/pom.xml @@ -61,14 +61,12 @@ governing permissions and limitations under the License. --> org.hamcrest hamcrest-all - - com.carrotsearch.randomizedtesting - randomizedtesting-runner - + org.apache.lucene lucene-test-framework + org.elasticsearch elasticsearch @@ -104,65 +102,10 @@ governing permissions and limitations under the License. --> 1.1.19 - - - - src/main/resources - true - - - - - src/test/resources - true - - plugin-test.properties - elasticsearch.yml - - - - src/test/resources - false - - log4j.xml - - - - - org.apache.maven.plugins - maven-compiler-plugin - - - com.carrotsearch.randomizedtesting - junit4-maven-plugin - - - org.apache.maven.plugins - maven-surefire-plugin - - - org.apache.maven.plugins - maven-source-plugin - org.apache.maven.plugins maven-assembly-plugin - - false - ${project.build.directory}/releases/ - - ${basedir}/src/main/assemblies/plugin.xml - - - - - package - - single - - - From d1945b6b2a65f3391af714ecbe574800ae5a2ced Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Thu, 21 May 2015 22:52:31 -0400 Subject: [PATCH 91/96] switch to plugin pom --- pom.xml | 31 +------------------------------ 1 file changed, 1 insertion(+), 30 deletions(-) diff --git a/pom.xml b/pom.xml index 3a3772c4cc4..19e32a18fee 100644 --- a/pom.xml +++ b/pom.xml @@ -39,7 +39,7 @@ governing permissions and limitations under the License. --> org.elasticsearch - elasticsearch-parent + elasticsearch-plugin 2.0.0-SNAPSHOT @@ -57,41 +57,12 @@ governing permissions and limitations under the License. --> - - org.hamcrest - hamcrest-all - - - - org.apache.lucene - lucene-test-framework - - - - org.elasticsearch - elasticsearch - provided - - com.google.apis google-api-services-compute ${google.gce.version} - - - - log4j - log4j - test - - - - org.elasticsearch - elasticsearch - test-jar - From a458bfc7a8b3d57419665a472eb4e4cf1bc6e762 Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Fri, 22 May 2015 00:10:40 -0400 Subject: [PATCH 92/96] remove tabs --- src/test/resources/log4j.xml | 38 ++++++++++++++++++------------------ 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/src/test/resources/log4j.xml b/src/test/resources/log4j.xml index 7804dbaf027..32b400ad8ee 100644 --- a/src/test/resources/log4j.xml +++ b/src/test/resources/log4j.xml @@ -13,27 +13,27 @@ governing permissions and limitations under the License. --> - - - - - - + + + + + + - - - + + + - - - - - - + + + + + + - - - - + + + + From 75e618d3a2e9d4d00ee5a652cef424eb243815fb Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Fri, 22 May 2015 09:25:15 -0400 Subject: [PATCH 93/96] remove logging properties --- src/test/resources/log4j.xml | 39 ------------------------------------ 1 file changed, 39 deletions(-) delete mode 100644 src/test/resources/log4j.xml diff --git a/src/test/resources/log4j.xml b/src/test/resources/log4j.xml deleted file mode 100644 index 32b400ad8ee..00000000000 --- a/src/test/resources/log4j.xml +++ /dev/null @@ -1,39 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - From 7efb36da5457f767963341916a707983897ba47c Mon Sep 17 00:00:00 2001 From: Michael McCandless Date: Tue, 26 May 2015 09:40:29 -0400 Subject: [PATCH 94/96] Absorb ImmutableSettings into Settings --- .../cloud/gce/AbstractGceTest.java | 3 +- .../discovery/gce/GceComputeEngineTest.java | 37 +++++++++---------- .../gce/itest/GceSimpleITest.java | 5 +-- 3 files changed, 21 insertions(+), 24 deletions(-) diff --git a/src/test/java/org/elasticsearch/cloud/gce/AbstractGceTest.java b/src/test/java/org/elasticsearch/cloud/gce/AbstractGceTest.java index b17199e415d..cc34726264a 100644 --- a/src/test/java/org/elasticsearch/cloud/gce/AbstractGceTest.java +++ b/src/test/java/org/elasticsearch/cloud/gce/AbstractGceTest.java @@ -20,7 +20,6 @@ package org.elasticsearch.cloud.gce; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.settings.ImmutableSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; import org.elasticsearch.env.FailedToResolveConfigException; @@ -36,7 +35,7 @@ public abstract class AbstractGceTest extends ElasticsearchIntegrationTest { @Override protected Settings nodeSettings(int nodeOrdinal) { - ImmutableSettings.Builder settings = ImmutableSettings.builder() + Settings.Builder settings = Settings.builder() .put(super.nodeSettings(nodeOrdinal)) .put("path.home", createTempDir()) .put("plugins." + PluginsService.LOAD_PLUGIN_FROM_CLASSPATH, true); diff --git a/src/test/java/org/elasticsearch/discovery/gce/GceComputeEngineTest.java b/src/test/java/org/elasticsearch/discovery/gce/GceComputeEngineTest.java index 35467c25c98..85de1dc482b 100644 --- a/src/test/java/org/elasticsearch/discovery/gce/GceComputeEngineTest.java +++ b/src/test/java/org/elasticsearch/discovery/gce/GceComputeEngineTest.java @@ -23,7 +23,6 @@ import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; import org.elasticsearch.cloud.gce.GceComputeService; import org.elasticsearch.cloud.gce.GceComputeService.Fields; import org.elasticsearch.common.collect.Lists; -import org.elasticsearch.common.settings.ImmutableSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.discovery.gce.mock.*; import org.elasticsearch.plugins.PluginsService; @@ -60,7 +59,7 @@ public class GceComputeEngineTest extends ElasticsearchIntegrationTest { } protected Settings settingsBuilder(int nodeOrdinal, Class mock, Settings settings) { - ImmutableSettings.Builder builder = ImmutableSettings.settingsBuilder() + Settings.Builder builder = Settings.settingsBuilder() .put("discovery.type", "gce") .put("cloud.gce.api.impl", mock) // We need the network to make the mock working @@ -94,10 +93,10 @@ public class GceComputeEngineTest extends ElasticsearchIntegrationTest { public void nodes_with_different_tags_and_no_tag_set() { startNode(1, GceComputeServiceTwoNodesDifferentTagsMock.class, - ImmutableSettings.EMPTY); + Settings.EMPTY); startNode(2, GceComputeServiceTwoNodesDifferentTagsMock.class, - ImmutableSettings.EMPTY); + Settings.EMPTY); // We expect having 2 nodes as part of the cluster, let's test that checkNumberOfNodes(2); @@ -113,10 +112,10 @@ public class GceComputeEngineTest extends ElasticsearchIntegrationTest { public void nodes_with_different_tags_and_one_tag_set() { startNode(1, GceComputeServiceTwoNodesDifferentTagsMock.class, - ImmutableSettings.settingsBuilder().put(Fields.TAGS, "elasticsearch").build()); + Settings.settingsBuilder().put(Fields.TAGS, "elasticsearch").build()); startNode(2, GceComputeServiceTwoNodesDifferentTagsMock.class, - ImmutableSettings.settingsBuilder().put(Fields.TAGS, "elasticsearch").build()); + Settings.settingsBuilder().put(Fields.TAGS, "elasticsearch").build()); // We expect having 1 nodes as part of the cluster, let's test that checkNumberOfNodes(1); @@ -132,10 +131,10 @@ public class GceComputeEngineTest extends ElasticsearchIntegrationTest { public void nodes_with_different_tags_and_two_tag_set() { startNode(1, GceComputeServiceTwoNodesDifferentTagsMock.class, - ImmutableSettings.settingsBuilder().put(Fields.TAGS, Lists.newArrayList("elasticsearch", "dev")).build()); + Settings.settingsBuilder().put(Fields.TAGS, Lists.newArrayList("elasticsearch", "dev")).build()); startNode(2, GceComputeServiceTwoNodesDifferentTagsMock.class, - ImmutableSettings.settingsBuilder().put(Fields.TAGS, Lists.newArrayList("elasticsearch", "dev")).build()); + Settings.settingsBuilder().put(Fields.TAGS, Lists.newArrayList("elasticsearch", "dev")).build()); // We expect having 1 nodes as part of the cluster, let's test that checkNumberOfNodes(1); @@ -145,10 +144,10 @@ public class GceComputeEngineTest extends ElasticsearchIntegrationTest { public void nodes_with_same_tags_and_no_tag_set() { startNode(1, GceComputeServiceTwoNodesSameTagsMock.class, - ImmutableSettings.EMPTY); + Settings.EMPTY); startNode(2, GceComputeServiceTwoNodesSameTagsMock.class, - ImmutableSettings.EMPTY); + Settings.EMPTY); // We expect having 2 nodes as part of the cluster, let's test that checkNumberOfNodes(2); @@ -158,10 +157,10 @@ public class GceComputeEngineTest extends ElasticsearchIntegrationTest { public void nodes_with_same_tags_and_one_tag_set() { startNode(1, GceComputeServiceTwoNodesSameTagsMock.class, - ImmutableSettings.settingsBuilder().put(Fields.TAGS, "elasticsearch").build()); + Settings.settingsBuilder().put(Fields.TAGS, "elasticsearch").build()); startNode(2, GceComputeServiceTwoNodesSameTagsMock.class, - ImmutableSettings.settingsBuilder().put(Fields.TAGS, "elasticsearch").build()); + Settings.settingsBuilder().put(Fields.TAGS, "elasticsearch").build()); // We expect having 2 nodes as part of the cluster, let's test that checkNumberOfNodes(2); @@ -171,10 +170,10 @@ public class GceComputeEngineTest extends ElasticsearchIntegrationTest { public void nodes_with_same_tags_and_two_tags_set() { startNode(1, GceComputeServiceTwoNodesSameTagsMock.class, - ImmutableSettings.settingsBuilder().put(Fields.TAGS, Lists.newArrayList("elasticsearch", "dev")).build()); + Settings.settingsBuilder().put(Fields.TAGS, Lists.newArrayList("elasticsearch", "dev")).build()); startNode(2, GceComputeServiceTwoNodesSameTagsMock.class, - ImmutableSettings.settingsBuilder().put(Fields.TAGS, Lists.newArrayList("elasticsearch", "dev")).build()); + Settings.settingsBuilder().put(Fields.TAGS, Lists.newArrayList("elasticsearch", "dev")).build()); // We expect having 2 nodes as part of the cluster, let's test that checkNumberOfNodes(2); @@ -184,11 +183,11 @@ public class GceComputeEngineTest extends ElasticsearchIntegrationTest { public void multiple_zones_and_two_nodes_in_same_zone() { startNode(1, GceComputeServiceTwoNodesOneZoneMock.class, - ImmutableSettings.settingsBuilder().put(Fields.ZONE, Lists.newArrayList("us-central1-a", "us-central1-b", + Settings.settingsBuilder().put(Fields.ZONE, Lists.newArrayList("us-central1-a", "us-central1-b", "us-central1-f", "europe-west1-a", "europe-west1-b")).build()); startNode(2, GceComputeServiceTwoNodesOneZoneMock.class, - ImmutableSettings.settingsBuilder().put(Fields.ZONE, Lists.newArrayList("us-central1-a", "us-central1-b", + Settings.settingsBuilder().put(Fields.ZONE, Lists.newArrayList("us-central1-a", "us-central1-b", "us-central1-f", "europe-west1-a", "europe-west1-b")).build()); // We expect having 2 nodes as part of the cluster, let's test that @@ -199,11 +198,11 @@ public class GceComputeEngineTest extends ElasticsearchIntegrationTest { public void multiple_zones_and_two_nodes_in_different_zones() { startNode(1, GceComputeServiceTwoNodesTwoZonesMock.class, - ImmutableSettings.settingsBuilder().put(Fields.ZONE, Lists.newArrayList("us-central1-a", "us-central1-b", + Settings.settingsBuilder().put(Fields.ZONE, Lists.newArrayList("us-central1-a", "us-central1-b", "us-central1-f", "europe-west1-a", "europe-west1-b")).build()); startNode(2, GceComputeServiceTwoNodesTwoZonesMock.class, - ImmutableSettings.settingsBuilder().put(Fields.ZONE, Lists.newArrayList("us-central1-a", "us-central1-b", + Settings.settingsBuilder().put(Fields.ZONE, Lists.newArrayList("us-central1-a", "us-central1-b", "us-central1-f", "europe-west1-a", "europe-west1-b")).build()); // We expect having 2 nodes as part of the cluster, let's test that @@ -217,7 +216,7 @@ public class GceComputeEngineTest extends ElasticsearchIntegrationTest { public void zero_node_43() { startNode(1, GceComputeServiceZeroNodeMock.class, - ImmutableSettings.settingsBuilder().put(Fields.ZONE, Lists.newArrayList("us-central1-a", "us-central1-b", + Settings.settingsBuilder().put(Fields.ZONE, Lists.newArrayList("us-central1-a", "us-central1-b", "us-central1-f", "europe-west1-a", "europe-west1-b")).build()); } diff --git a/src/test/java/org/elasticsearch/gce/itest/GceSimpleITest.java b/src/test/java/org/elasticsearch/gce/itest/GceSimpleITest.java index 896f98a76c4..67ecbee8592 100644 --- a/src/test/java/org/elasticsearch/gce/itest/GceSimpleITest.java +++ b/src/test/java/org/elasticsearch/gce/itest/GceSimpleITest.java @@ -21,7 +21,6 @@ package org.elasticsearch.gce.itest; import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; import org.elasticsearch.cloud.gce.AbstractGceTest; -import org.elasticsearch.common.settings.ImmutableSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.test.ElasticsearchIntegrationTest; @@ -43,7 +42,7 @@ public class GceSimpleITest extends AbstractGceTest { @Override protected Settings nodeSettings(int nodeOrdinal) { - return ImmutableSettings.builder() + return Settings.builder() .put(super.nodeSettings(nodeOrdinal)) .put("plugins." + PluginsService.LOAD_PLUGIN_FROM_CLASSPATH, true) .build(); @@ -62,7 +61,7 @@ public class GceSimpleITest extends AbstractGceTest { public Settings indexSettings() { // During restore we frequently restore index to exactly the same state it was before, that might cause the same // checksum file to be written twice during restore operation - return ImmutableSettings.builder().put(super.indexSettings()) + return Settings.builder().put(super.indexSettings()) .build(); } } From 7d67dfd61d5d6f75808fd17bc6b00d08fb786a29 Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Thu, 4 Jun 2015 16:27:29 -0400 Subject: [PATCH 95/96] Don't use shaded APIs --- .../cloud/gce/GceComputeServiceImpl.java | 11 ++++++----- .../discovery/gce/GceUnicastHostsProvider.java | 7 ++++--- .../plugin/cloud/gce/CloudGcePlugin.java | 6 +++--- .../discovery/gce/GceComputeEngineTest.java | 2 +- .../gce/mock/GceComputeServiceAbstractMock.java | 4 ++-- ...GceComputeServiceTwoNodesDifferentTagsMock.java | 12 ++++++------ .../mock/GceComputeServiceTwoNodesOneZoneMock.java | 8 ++++---- .../GceComputeServiceTwoNodesSameTagsMock.java | 12 ++++++------ .../GceComputeServiceTwoNodesTwoZonesMock.java | 8 ++++---- .../gce/mock/GceComputeServiceZeroNodeMock.java | 14 +++++++------- 10 files changed, 43 insertions(+), 41 deletions(-) diff --git a/src/main/java/org/elasticsearch/cloud/gce/GceComputeServiceImpl.java b/src/main/java/org/elasticsearch/cloud/gce/GceComputeServiceImpl.java index a786041c31f..ceb3b8633d2 100644 --- a/src/main/java/org/elasticsearch/cloud/gce/GceComputeServiceImpl.java +++ b/src/main/java/org/elasticsearch/cloud/gce/GceComputeServiceImpl.java @@ -27,10 +27,10 @@ import com.google.api.client.json.jackson2.JacksonFactory; import com.google.api.services.compute.Compute; import com.google.api.services.compute.model.Instance; import com.google.api.services.compute.model.InstanceList; +import com.google.common.base.Function; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.common.base.Function; -import org.elasticsearch.common.collect.Iterables; -import org.elasticsearch.common.collect.Lists; import org.elasticsearch.common.component.AbstractLifecycleComponent; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; @@ -38,6 +38,7 @@ import org.elasticsearch.common.settings.SettingsFilter; import org.elasticsearch.common.unit.TimeValue; import java.io.IOException; +import java.util.ArrayList; import java.util.Collection; import java.util.List; @@ -67,7 +68,7 @@ public class GceComputeServiceImpl extends AbstractLifecycleComponent(); String ipAddress = null; try { InetAddress inetAddress = networkService.resolvePublishHostAddress(null); diff --git a/src/main/java/org/elasticsearch/plugin/cloud/gce/CloudGcePlugin.java b/src/main/java/org/elasticsearch/plugin/cloud/gce/CloudGcePlugin.java index b6f3cf0216f..0333db58665 100644 --- a/src/main/java/org/elasticsearch/plugin/cloud/gce/CloudGcePlugin.java +++ b/src/main/java/org/elasticsearch/plugin/cloud/gce/CloudGcePlugin.java @@ -20,12 +20,12 @@ package org.elasticsearch.plugin.cloud.gce; import org.elasticsearch.cloud.gce.GceModule; -import org.elasticsearch.common.collect.Lists; import org.elasticsearch.common.component.LifecycleComponent; import org.elasticsearch.common.inject.Module; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.plugins.AbstractPlugin; +import java.util.ArrayList; import java.util.Collection; /** @@ -51,7 +51,7 @@ public class CloudGcePlugin extends AbstractPlugin { @Override public Collection> modules() { - Collection> modules = Lists.newArrayList(); + Collection> modules = new ArrayList<>(); if (settings.getAsBoolean("cloud.enabled", true)) { modules.add(GceModule.class); } @@ -60,7 +60,7 @@ public class CloudGcePlugin extends AbstractPlugin { @Override public Collection> services() { - Collection> services = Lists.newArrayList(); + Collection> services = new ArrayList<>(); if (settings.getAsBoolean("cloud.enabled", true)) { // services.add(GceComputeServiceImpl.class); } diff --git a/src/test/java/org/elasticsearch/discovery/gce/GceComputeEngineTest.java b/src/test/java/org/elasticsearch/discovery/gce/GceComputeEngineTest.java index 85de1dc482b..2bbfa60cfe1 100644 --- a/src/test/java/org/elasticsearch/discovery/gce/GceComputeEngineTest.java +++ b/src/test/java/org/elasticsearch/discovery/gce/GceComputeEngineTest.java @@ -19,10 +19,10 @@ package org.elasticsearch.discovery.gce; +import com.google.common.collect.Lists; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; import org.elasticsearch.cloud.gce.GceComputeService; import org.elasticsearch.cloud.gce.GceComputeService.Fields; -import org.elasticsearch.common.collect.Lists; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.discovery.gce.mock.*; import org.elasticsearch.plugins.PluginsService; diff --git a/src/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceAbstractMock.java b/src/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceAbstractMock.java index 7b5a19461b1..e885162c22f 100644 --- a/src/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceAbstractMock.java +++ b/src/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceAbstractMock.java @@ -23,9 +23,9 @@ import com.google.api.services.compute.model.Instance; import com.google.api.services.compute.model.Metadata; import com.google.api.services.compute.model.NetworkInterface; import com.google.api.services.compute.model.Tags; +import com.google.common.collect.Lists; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.cloud.gce.GceComputeService; -import org.elasticsearch.common.collect.Lists; import org.elasticsearch.common.component.AbstractLifecycleComponent; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.discovery.gce.GceComputeEngineTest; @@ -42,7 +42,7 @@ import static com.carrotsearch.randomizedtesting.RandomizedTest.randomInt; public abstract class GceComputeServiceAbstractMock extends AbstractLifecycleComponent implements GceComputeService { - protected abstract List> getTags(); + protected abstract List> getTags(); protected abstract List getZones(); private final List zoneList; diff --git a/src/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceTwoNodesDifferentTagsMock.java b/src/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceTwoNodesDifferentTagsMock.java index c261e2ea3d8..bba393c6540 100644 --- a/src/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceTwoNodesDifferentTagsMock.java +++ b/src/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceTwoNodesDifferentTagsMock.java @@ -19,29 +19,29 @@ package org.elasticsearch.discovery.gce.mock; -import org.elasticsearch.common.collect.Lists; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; /** * */ public class GceComputeServiceTwoNodesDifferentTagsMock extends GceComputeServiceAbstractMock { - private static List> tags = Lists.newArrayList( - Lists.newArrayList("dev"), - Lists.newArrayList("elasticsearch","dev")); + private static List> tags = Arrays.asList( + Arrays.asList("dev"), + Arrays.asList("elasticsearch", "dev")); @Override - protected List> getTags() { + protected List> getTags() { return tags; } @Override protected List getZones() { - return Lists.newArrayList(); + return new ArrayList(); } @Inject diff --git a/src/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceTwoNodesOneZoneMock.java b/src/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceTwoNodesOneZoneMock.java index 5a87c6fcbf6..9221b0263f1 100644 --- a/src/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceTwoNodesOneZoneMock.java +++ b/src/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceTwoNodesOneZoneMock.java @@ -19,22 +19,22 @@ package org.elasticsearch.discovery.gce.mock; -import org.elasticsearch.common.collect.Lists; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; /** * */ public class GceComputeServiceTwoNodesOneZoneMock extends GceComputeServiceAbstractMock { - private static List zones = Lists.newArrayList("us-central1-a","us-central1-a"); + private static List zones = Arrays.asList("us-central1-a", "us-central1-a"); @Override - protected List> getTags() { - return Lists.newArrayList(); + protected List> getTags() { + return new ArrayList<>(); } @Override diff --git a/src/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceTwoNodesSameTagsMock.java b/src/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceTwoNodesSameTagsMock.java index 4de658f284a..e2384a6a441 100644 --- a/src/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceTwoNodesSameTagsMock.java +++ b/src/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceTwoNodesSameTagsMock.java @@ -19,30 +19,30 @@ package org.elasticsearch.discovery.gce.mock; -import org.elasticsearch.common.collect.Lists; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; /** * */ public class GceComputeServiceTwoNodesSameTagsMock extends GceComputeServiceAbstractMock { - private static List> tags = Lists.newArrayList( - Lists.newArrayList("elasticsearch","dev"), - Lists.newArrayList("elasticsearch","dev")); + private static List> tags = Arrays.asList( + Arrays.asList("elasticsearch", "dev"), + Arrays.asList("elasticsearch", "dev")); @Override - protected List> getTags() { + protected List> getTags() { return tags; } @Override protected List getZones() { - return Lists.newArrayList(); + return new ArrayList<>(); } @Inject diff --git a/src/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceTwoNodesTwoZonesMock.java b/src/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceTwoNodesTwoZonesMock.java index 527e508d844..a1a5cbeb864 100644 --- a/src/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceTwoNodesTwoZonesMock.java +++ b/src/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceTwoNodesTwoZonesMock.java @@ -19,22 +19,22 @@ package org.elasticsearch.discovery.gce.mock; -import org.elasticsearch.common.collect.Lists; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; /** * */ public class GceComputeServiceTwoNodesTwoZonesMock extends GceComputeServiceAbstractMock { - private static List zones = Lists.newArrayList("us-central1-a","europe-west1-a"); + private static List zones = Arrays.asList("us-central1-a", "europe-west1-a"); @Override - protected List> getTags() { - return Lists.newArrayList(); + protected List> getTags() { + return new ArrayList(); } @Override diff --git a/src/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceZeroNodeMock.java b/src/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceZeroNodeMock.java index 6fd22a47c0b..7eccfd0f8bd 100644 --- a/src/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceZeroNodeMock.java +++ b/src/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceZeroNodeMock.java @@ -20,9 +20,9 @@ package org.elasticsearch.discovery.gce.mock; import com.google.api.services.compute.model.Instance; -import org.elasticsearch.common.base.Function; -import org.elasticsearch.common.collect.Iterables; -import org.elasticsearch.common.collect.Lists; +import com.google.common.base.Function; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; @@ -36,13 +36,13 @@ import java.util.List; public class GceComputeServiceZeroNodeMock extends GceComputeServiceAbstractMock { @Override - protected List> getTags() { - return Lists.newArrayList(); + protected List> getTags() { + return new ArrayList(); } @Override protected List getZones() { - return Lists.newArrayList(); + return new ArrayList(); } private final List zoneList; @@ -55,7 +55,7 @@ public class GceComputeServiceZeroNodeMock extends GceComputeServiceAbstractMock @Override public List apply(String zoneId) { // If we return null here we will get a trace as explained in issue 43 - return Lists.newArrayList(); + return new ArrayList(); } }); From 4e9e74da97b37e712a25da8c415760bb92ee9802 Mon Sep 17 00:00:00 2001 From: Simon Willnauer Date: Fri, 5 Jun 2015 13:12:28 +0200 Subject: [PATCH 96/96] add cloud-gce module --- .gitignore | 13 -- CONTRIBUTING.md | 98 --------- LICENSE.txt | 202 ------------------ dev-tools/release.py | 134 ------------ NOTICE.txt => plugins/cloud-gce/NOTICE.txt | 0 README.md => plugins/cloud-gce/README.md | 0 pom.xml => plugins/cloud-gce/pom.xml | 23 +- .../cloud-gce/src}/main/assemblies/plugin.xml | 0 .../cloud/gce/GceComputeService.java | 0 .../cloud/gce/GceComputeServiceImpl.java | 0 .../elasticsearch/cloud/gce/GceModule.java | 0 .../discovery/gce/GceDiscovery.java | 0 .../discovery/gce/GceDiscoveryModule.java | 0 .../gce/GceUnicastHostsProvider.java | 0 .../plugin/cloud/gce/CloudGcePlugin.java | 0 .../src}/main/resources/es-plugin.properties | 0 .../cloud/gce/AbstractGceTest.java | 0 .../discovery/gce/GceComputeEngineTest.java | 0 .../discovery/gce/PropertiesHelper.java | 0 .../mock/GceComputeServiceAbstractMock.java | 0 ...mputeServiceTwoNodesDifferentTagsMock.java | 0 .../GceComputeServiceTwoNodesOneZoneMock.java | 0 ...GceComputeServiceTwoNodesSameTagsMock.java | 0 ...GceComputeServiceTwoNodesTwoZonesMock.java | 0 .../mock/GceComputeServiceZeroNodeMock.java | 0 .../gce/itest/GceSimpleITest.java | 0 .../test/resources/plugin-test.properties | 0 27 files changed, 2 insertions(+), 468 deletions(-) delete mode 100644 .gitignore delete mode 100644 CONTRIBUTING.md delete mode 100644 LICENSE.txt delete mode 100644 dev-tools/release.py rename NOTICE.txt => plugins/cloud-gce/NOTICE.txt (100%) rename README.md => plugins/cloud-gce/README.md (100%) rename pom.xml => plugins/cloud-gce/pom.xml (73%) rename {src => plugins/cloud-gce/src}/main/assemblies/plugin.xml (100%) rename {src => plugins/cloud-gce/src}/main/java/org/elasticsearch/cloud/gce/GceComputeService.java (100%) rename {src => plugins/cloud-gce/src}/main/java/org/elasticsearch/cloud/gce/GceComputeServiceImpl.java (100%) rename {src => plugins/cloud-gce/src}/main/java/org/elasticsearch/cloud/gce/GceModule.java (100%) rename {src => plugins/cloud-gce/src}/main/java/org/elasticsearch/discovery/gce/GceDiscovery.java (100%) rename {src => plugins/cloud-gce/src}/main/java/org/elasticsearch/discovery/gce/GceDiscoveryModule.java (100%) rename {src => plugins/cloud-gce/src}/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java (100%) rename {src => plugins/cloud-gce/src}/main/java/org/elasticsearch/plugin/cloud/gce/CloudGcePlugin.java (100%) rename {src => plugins/cloud-gce/src}/main/resources/es-plugin.properties (100%) rename {src => plugins/cloud-gce/src}/test/java/org/elasticsearch/cloud/gce/AbstractGceTest.java (100%) rename {src => plugins/cloud-gce/src}/test/java/org/elasticsearch/discovery/gce/GceComputeEngineTest.java (100%) rename {src => plugins/cloud-gce/src}/test/java/org/elasticsearch/discovery/gce/PropertiesHelper.java (100%) rename {src => plugins/cloud-gce/src}/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceAbstractMock.java (100%) rename {src => plugins/cloud-gce/src}/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceTwoNodesDifferentTagsMock.java (100%) rename {src => plugins/cloud-gce/src}/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceTwoNodesOneZoneMock.java (100%) rename {src => plugins/cloud-gce/src}/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceTwoNodesSameTagsMock.java (100%) rename {src => plugins/cloud-gce/src}/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceTwoNodesTwoZonesMock.java (100%) rename {src => plugins/cloud-gce/src}/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceZeroNodeMock.java (100%) rename {src => plugins/cloud-gce/src}/test/java/org/elasticsearch/gce/itest/GceSimpleITest.java (100%) rename {src => plugins/cloud-gce/src}/test/resources/plugin-test.properties (100%) diff --git a/.gitignore b/.gitignore deleted file mode 100644 index 9533848e238..00000000000 --- a/.gitignore +++ /dev/null @@ -1,13 +0,0 @@ -/data -/work -/logs -/.idea -/target -.DS_Store -*.iml -/.project -/.settings -/.classpath -/plugin_tools -/.local-execution-hints.log -/.local-*-execution-hints.log diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md deleted file mode 100644 index 54c5c543c89..00000000000 --- a/CONTRIBUTING.md +++ /dev/null @@ -1,98 +0,0 @@ -Contributing to elasticsearch -============================= - -Elasticsearch is an open source project and we love to receive contributions from our community — you! There are many ways to contribute, from writing tutorials or blog posts, improving the documentation, submitting bug reports and feature requests or writing code which can be incorporated into Elasticsearch itself. - -Bug reports ------------ - -If you think you have found a bug in Elasticsearch, first make sure that you are testing against the [latest version of Elasticsearch](http://www.elasticsearch.org/download/) - your issue may already have been fixed. If not, search our [issues list](https://github.com/elasticsearch/elasticsearch/issues) on GitHub in case a similar issue has already been opened. - -It is very helpful if you can prepare a reproduction of the bug. In other words, provide a small test case which we can run to confirm your bug. It makes it easier to find the problem and to fix it. Test cases should be provided as `curl` commands which we can copy and paste into a terminal to run it locally, for example: - -```sh -# delete the index -curl -XDELETE localhost:9200/test - -# insert a document -curl -XPUT localhost:9200/test/test/1 -d '{ - "title": "test document" -}' - -# this should return XXXX but instead returns YYY -curl .... -``` - -Provide as much information as you can. You may think that the problem lies with your query, when actually it depends on how your data is indexed. The easier it is for us to recreate your problem, the faster it is likely to be fixed. - -Feature requests ----------------- - -If you find yourself wishing for a feature that doesn't exist in Elasticsearch, you are probably not alone. There are bound to be others out there with similar needs. Many of the features that Elasticsearch has today have been added because our users saw the need. -Open an issue on our [issues list](https://github.com/elasticsearch/elasticsearch/issues) on GitHub which describes the feature you would like to see, why you need it, and how it should work. - -Contributing code and documentation changes -------------------------------------------- - -If you have a bugfix or new feature that you would like to contribute to Elasticsearch, please find or open an issue about it first. Talk about what you would like to do. It may be that somebody is already working on it, or that there are particular issues that you should know about before implementing the change. - -We enjoy working with contributors to get their code accepted. There are many approaches to fixing a problem and it is important to find the best approach before writing too much code. - -The process for contributing to any of the [Elasticsearch repositories](https://github.com/elasticsearch/) is similar. Details for individual projects can be found below. - -### Fork and clone the repository - -You will need to fork the main Elasticsearch code or documentation repository and clone it to your local machine. See -[github help page](https://help.github.com/articles/fork-a-repo) for help. - -Further instructions for specific projects are given below. - -### Submitting your changes - -Once your changes and tests are ready to submit for review: - -1. Test your changes -Run the test suite to make sure that nothing is broken. - -2. Sign the Contributor License Agreement -Please make sure you have signed our [Contributor License Agreement](http://www.elasticsearch.org/contributor-agreement/). We are not asking you to assign copyright to us, but to give us the right to distribute your code without restriction. We ask this of all contributors in order to assure our users of the origin and continuing existence of the code. You only need to sign the CLA once. - -3. Rebase your changes -Update your local repository with the most recent code from the main Elasticsearch repository, and rebase your branch on top of the latest master branch. We prefer your changes to be squashed into a single commit. - -4. Submit a pull request -Push your local changes to your forked copy of the repository and [submit a pull request](https://help.github.com/articles/using-pull-requests). In the pull request, describe what your changes do and mention the number of the issue where discussion has taken place, eg "Closes #123". - -Then sit back and wait. There will probably be discussion about the pull request and, if any changes are needed, we would love to work with you to get your pull request merged into Elasticsearch. - - -Contributing to the Elasticsearch plugin ----------------------------------------- - -**Repository:** [https://github.com/elasticsearch/elasticsearch-cloud-gce](https://github.com/elasticsearch/elasticsearch-cloud-gce) - -Make sure you have [Maven](http://maven.apache.org) installed, as Elasticsearch uses it as its build system. Integration with IntelliJ and Eclipse should work out of the box. Eclipse users can automatically configure their IDE by running `mvn eclipse:eclipse` and then importing the project into their workspace: `File > Import > Existing project into workspace`. - -Please follow these formatting guidelines: - -* Java indent is 4 spaces -* Line width is 140 characters -* The rest is left to Java coding standards -* Disable “auto-format on save” to prevent unnecessary format changes. This makes reviews much harder as it generates unnecessary formatting changes. If your IDE supports formatting only modified chunks that is fine to do. - -To create a distribution from the source, simply run: - -```sh -cd elasticsearch-cloud-gce/ -mvn clean package -DskipTests -``` - -You will find the newly built packages under: `./target/releases/`. - -Before submitting your changes, run the test suite to make sure that nothing is broken, with: - -```sh -mvn clean test -``` - -Source: [Contributing to elasticsearch](http://www.elasticsearch.org/contributing-to-elasticsearch/) diff --git a/LICENSE.txt b/LICENSE.txt deleted file mode 100644 index d6456956733..00000000000 --- a/LICENSE.txt +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/dev-tools/release.py b/dev-tools/release.py deleted file mode 100644 index edcc637d068..00000000000 --- a/dev-tools/release.py +++ /dev/null @@ -1,134 +0,0 @@ -# Licensed to Elasticsearch under one or more contributor -# license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright -# ownership. Elasticsearch licenses this file to you under -# the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on -# an 'AS IS' BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -# either express or implied. See the License for the specific -# language governing permissions and limitations under the License. - -import datetime -import os -import shutil -import sys -import time -import urllib -import urllib.request -import zipfile - -from os.path import dirname, abspath - -""" - This tool builds a release from the a given elasticsearch plugin branch. - - It is basically a wrapper on top of launch_release.py which: - - - tries to get a more recent version of launch_release.py in ... - - download it if needed - - launch it passing all arguments to it, like: - - $ python3 dev_tools/release.py --branch master --publish --remote origin - - Important options: - - # Dry run - $ python3 dev_tools/release.py - - # Dry run without tests - python3 dev_tools/release.py --skiptests - - # Release, publish artifacts and announce - $ python3 dev_tools/release.py --publish - - See full documentation in launch_release.py -""" -env = os.environ - -# Change this if the source repository for your scripts is at a different location -SOURCE_REPO = 'elasticsearch/elasticsearch-plugins-script' -# We define that we should download again the script after 1 days -SCRIPT_OBSOLETE_DAYS = 1 -# We ignore in master.zip file the following files -IGNORED_FILES = ['.gitignore', 'README.md'] - - -ROOT_DIR = abspath(os.path.join(abspath(dirname(__file__)), '../')) -TARGET_TOOLS_DIR = ROOT_DIR + '/plugin_tools' -DEV_TOOLS_DIR = ROOT_DIR + '/dev-tools' -BUILD_RELEASE_FILENAME = 'release.zip' -BUILD_RELEASE_FILE = TARGET_TOOLS_DIR + '/' + BUILD_RELEASE_FILENAME -SOURCE_URL = 'https://github.com/%s/archive/master.zip' % SOURCE_REPO - -# Download a recent version of the release plugin tool -try: - os.mkdir(TARGET_TOOLS_DIR) - print('directory %s created' % TARGET_TOOLS_DIR) -except FileExistsError: - pass - - -try: - # we check latest update. If we ran an update recently, we - # are not going to check it again - download = True - - try: - last_download_time = datetime.datetime.fromtimestamp(os.path.getmtime(BUILD_RELEASE_FILE)) - if (datetime.datetime.now()-last_download_time).days < SCRIPT_OBSOLETE_DAYS: - download = False - except FileNotFoundError: - pass - - if download: - urllib.request.urlretrieve(SOURCE_URL, BUILD_RELEASE_FILE) - with zipfile.ZipFile(BUILD_RELEASE_FILE) as myzip: - for member in myzip.infolist(): - filename = os.path.basename(member.filename) - # skip directories - if not filename: - continue - if filename in IGNORED_FILES: - continue - - # copy file (taken from zipfile's extract) - source = myzip.open(member.filename) - target = open(os.path.join(TARGET_TOOLS_DIR, filename), "wb") - with source, target: - shutil.copyfileobj(source, target) - # We keep the original date - date_time = time.mktime(member.date_time + (0, 0, -1)) - os.utime(os.path.join(TARGET_TOOLS_DIR, filename), (date_time, date_time)) - print('plugin-tools updated from %s' % SOURCE_URL) -except urllib.error.HTTPError: - pass - - -# Let see if we need to update the release.py script itself -source_time = os.path.getmtime(TARGET_TOOLS_DIR + '/release.py') -repo_time = os.path.getmtime(DEV_TOOLS_DIR + '/release.py') -if source_time > repo_time: - input('release.py needs an update. Press a key to update it...') - shutil.copyfile(TARGET_TOOLS_DIR + '/release.py', DEV_TOOLS_DIR + '/release.py') - -# We can launch the build process -try: - PYTHON = 'python' - # make sure python3 is used if python3 is available - # some systems use python 2 as default - os.system('python3 --version > /dev/null 2>&1') - PYTHON = 'python3' -except RuntimeError: - pass - -release_args = '' -for x in range(1, len(sys.argv)): - release_args += ' ' + sys.argv[x] - -os.system('%s %s/build_release.py %s' % (PYTHON, TARGET_TOOLS_DIR, release_args)) diff --git a/NOTICE.txt b/plugins/cloud-gce/NOTICE.txt similarity index 100% rename from NOTICE.txt rename to plugins/cloud-gce/NOTICE.txt diff --git a/README.md b/plugins/cloud-gce/README.md similarity index 100% rename from README.md rename to plugins/cloud-gce/README.md diff --git a/pom.xml b/plugins/cloud-gce/pom.xml similarity index 73% rename from pom.xml rename to plugins/cloud-gce/pom.xml index 19e32a18fee..ca9206abccc 100644 --- a/pom.xml +++ b/plugins/cloud-gce/pom.xml @@ -15,27 +15,12 @@ governing permissions and limitations under the License. --> xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - org.elasticsearch + org.elasticsearch.plugin elasticsearch-cloud-gce - 3.0.0-SNAPSHOT + Elasticsearch Google Compute Engine cloud plugin The Google Compute Engine (GCE) Cloud plugin allows to use GCE API for the unicast discovery mechanism. - https://github.com/elastic/elasticsearch-cloud-gce/ - 2013 - - - - The Apache Software License, Version 2.0 - http://www.apache.org/licenses/LICENSE-2.0.txt - repo - - - - scm:git:git@github.com:elastic/elasticsearch-cloud-gce.git - scm:git:git@github.com:elastic/elasticsearch-cloud-gce.git - http://github.com/elastic/elasticsearch-cloud-gce - org.elasticsearch @@ -43,10 +28,6 @@ governing permissions and limitations under the License. --> 2.0.0-SNAPSHOT - - GitHub - https://github.com/elastic/elasticsearch-cloud-gce/issues/ - v1-rev59-1.20.0 diff --git a/src/main/assemblies/plugin.xml b/plugins/cloud-gce/src/main/assemblies/plugin.xml similarity index 100% rename from src/main/assemblies/plugin.xml rename to plugins/cloud-gce/src/main/assemblies/plugin.xml diff --git a/src/main/java/org/elasticsearch/cloud/gce/GceComputeService.java b/plugins/cloud-gce/src/main/java/org/elasticsearch/cloud/gce/GceComputeService.java similarity index 100% rename from src/main/java/org/elasticsearch/cloud/gce/GceComputeService.java rename to plugins/cloud-gce/src/main/java/org/elasticsearch/cloud/gce/GceComputeService.java diff --git a/src/main/java/org/elasticsearch/cloud/gce/GceComputeServiceImpl.java b/plugins/cloud-gce/src/main/java/org/elasticsearch/cloud/gce/GceComputeServiceImpl.java similarity index 100% rename from src/main/java/org/elasticsearch/cloud/gce/GceComputeServiceImpl.java rename to plugins/cloud-gce/src/main/java/org/elasticsearch/cloud/gce/GceComputeServiceImpl.java diff --git a/src/main/java/org/elasticsearch/cloud/gce/GceModule.java b/plugins/cloud-gce/src/main/java/org/elasticsearch/cloud/gce/GceModule.java similarity index 100% rename from src/main/java/org/elasticsearch/cloud/gce/GceModule.java rename to plugins/cloud-gce/src/main/java/org/elasticsearch/cloud/gce/GceModule.java diff --git a/src/main/java/org/elasticsearch/discovery/gce/GceDiscovery.java b/plugins/cloud-gce/src/main/java/org/elasticsearch/discovery/gce/GceDiscovery.java similarity index 100% rename from src/main/java/org/elasticsearch/discovery/gce/GceDiscovery.java rename to plugins/cloud-gce/src/main/java/org/elasticsearch/discovery/gce/GceDiscovery.java diff --git a/src/main/java/org/elasticsearch/discovery/gce/GceDiscoveryModule.java b/plugins/cloud-gce/src/main/java/org/elasticsearch/discovery/gce/GceDiscoveryModule.java similarity index 100% rename from src/main/java/org/elasticsearch/discovery/gce/GceDiscoveryModule.java rename to plugins/cloud-gce/src/main/java/org/elasticsearch/discovery/gce/GceDiscoveryModule.java diff --git a/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java b/plugins/cloud-gce/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java similarity index 100% rename from src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java rename to plugins/cloud-gce/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java diff --git a/src/main/java/org/elasticsearch/plugin/cloud/gce/CloudGcePlugin.java b/plugins/cloud-gce/src/main/java/org/elasticsearch/plugin/cloud/gce/CloudGcePlugin.java similarity index 100% rename from src/main/java/org/elasticsearch/plugin/cloud/gce/CloudGcePlugin.java rename to plugins/cloud-gce/src/main/java/org/elasticsearch/plugin/cloud/gce/CloudGcePlugin.java diff --git a/src/main/resources/es-plugin.properties b/plugins/cloud-gce/src/main/resources/es-plugin.properties similarity index 100% rename from src/main/resources/es-plugin.properties rename to plugins/cloud-gce/src/main/resources/es-plugin.properties diff --git a/src/test/java/org/elasticsearch/cloud/gce/AbstractGceTest.java b/plugins/cloud-gce/src/test/java/org/elasticsearch/cloud/gce/AbstractGceTest.java similarity index 100% rename from src/test/java/org/elasticsearch/cloud/gce/AbstractGceTest.java rename to plugins/cloud-gce/src/test/java/org/elasticsearch/cloud/gce/AbstractGceTest.java diff --git a/src/test/java/org/elasticsearch/discovery/gce/GceComputeEngineTest.java b/plugins/cloud-gce/src/test/java/org/elasticsearch/discovery/gce/GceComputeEngineTest.java similarity index 100% rename from src/test/java/org/elasticsearch/discovery/gce/GceComputeEngineTest.java rename to plugins/cloud-gce/src/test/java/org/elasticsearch/discovery/gce/GceComputeEngineTest.java diff --git a/src/test/java/org/elasticsearch/discovery/gce/PropertiesHelper.java b/plugins/cloud-gce/src/test/java/org/elasticsearch/discovery/gce/PropertiesHelper.java similarity index 100% rename from src/test/java/org/elasticsearch/discovery/gce/PropertiesHelper.java rename to plugins/cloud-gce/src/test/java/org/elasticsearch/discovery/gce/PropertiesHelper.java diff --git a/src/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceAbstractMock.java b/plugins/cloud-gce/src/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceAbstractMock.java similarity index 100% rename from src/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceAbstractMock.java rename to plugins/cloud-gce/src/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceAbstractMock.java diff --git a/src/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceTwoNodesDifferentTagsMock.java b/plugins/cloud-gce/src/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceTwoNodesDifferentTagsMock.java similarity index 100% rename from src/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceTwoNodesDifferentTagsMock.java rename to plugins/cloud-gce/src/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceTwoNodesDifferentTagsMock.java diff --git a/src/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceTwoNodesOneZoneMock.java b/plugins/cloud-gce/src/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceTwoNodesOneZoneMock.java similarity index 100% rename from src/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceTwoNodesOneZoneMock.java rename to plugins/cloud-gce/src/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceTwoNodesOneZoneMock.java diff --git a/src/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceTwoNodesSameTagsMock.java b/plugins/cloud-gce/src/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceTwoNodesSameTagsMock.java similarity index 100% rename from src/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceTwoNodesSameTagsMock.java rename to plugins/cloud-gce/src/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceTwoNodesSameTagsMock.java diff --git a/src/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceTwoNodesTwoZonesMock.java b/plugins/cloud-gce/src/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceTwoNodesTwoZonesMock.java similarity index 100% rename from src/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceTwoNodesTwoZonesMock.java rename to plugins/cloud-gce/src/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceTwoNodesTwoZonesMock.java diff --git a/src/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceZeroNodeMock.java b/plugins/cloud-gce/src/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceZeroNodeMock.java similarity index 100% rename from src/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceZeroNodeMock.java rename to plugins/cloud-gce/src/test/java/org/elasticsearch/discovery/gce/mock/GceComputeServiceZeroNodeMock.java diff --git a/src/test/java/org/elasticsearch/gce/itest/GceSimpleITest.java b/plugins/cloud-gce/src/test/java/org/elasticsearch/gce/itest/GceSimpleITest.java similarity index 100% rename from src/test/java/org/elasticsearch/gce/itest/GceSimpleITest.java rename to plugins/cloud-gce/src/test/java/org/elasticsearch/gce/itest/GceSimpleITest.java diff --git a/src/test/resources/plugin-test.properties b/plugins/cloud-gce/src/test/resources/plugin-test.properties similarity index 100% rename from src/test/resources/plugin-test.properties rename to plugins/cloud-gce/src/test/resources/plugin-test.properties