diff --git a/core/src/main/java/org/elasticsearch/common/network/NetworkService.java b/core/src/main/java/org/elasticsearch/common/network/NetworkService.java index 8eff70e7bd8..cd46d1416f4 100644 --- a/core/src/main/java/org/elasticsearch/common/network/NetworkService.java +++ b/core/src/main/java/org/elasticsearch/common/network/NetworkService.java @@ -27,7 +27,6 @@ import org.elasticsearch.common.unit.TimeValue; import java.io.IOException; import java.net.InetAddress; -import java.net.UnknownHostException; import java.util.List; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.TimeUnit; @@ -73,7 +72,7 @@ public class NetworkService extends AbstractComponent { /** * Resolves a custom value handling, return null if can't handle it. */ - InetAddress[] resolveIfPossible(String value); + InetAddress[] resolveIfPossible(String value) throws IOException; } private final List customNameResolvers = new CopyOnWriteArrayList<>(); @@ -162,7 +161,7 @@ public class NetworkService extends AbstractComponent { return address; } - private InetAddress[] resolveInetAddress(String host) throws UnknownHostException, IOException { + private InetAddress[] resolveInetAddress(String host) throws IOException { if ((host.startsWith("#") && host.endsWith("#")) || (host.startsWith("_") && host.endsWith("_"))) { host = host.substring(1, host.length() - 1); // allow custom resolvers to have special names diff --git a/docs/plugins/cloud-gce.asciidoc b/docs/plugins/cloud-gce.asciidoc index 3f5b9da45d0..5cb241ed505 100644 --- a/docs/plugins/cloud-gce.asciidoc +++ b/docs/plugins/cloud-gce.asciidoc @@ -46,6 +46,44 @@ discovery: type: gce -------------------------------------------------- + +[IMPORTANT] +.Binding the network host +============================================== + +It's important to define `network.host` as by default it's bound to `localhost`. + +You can use {ref}/modules-network.html[core network host settings] or +<>: + +============================================== + +[[discovery-gce-network-host]] +==== GCE Network Host + +When the `cloud-gce` plugin is installed, the following are also allowed +as valid network host settings: + +[cols="<,<",options="header",] +|================================================================== +|GCE Host Value |Description +|`_gce:privateIp:X_` |The private IP address of the machine for a given network interface. +|`_gce:hostname_` |The hostname of the machine. +|`_gce_` |Same as `_gce:privateIp:0_` (recommended). +|================================================================== + +Examples: + +[source,yaml] +-------------------------------------------------- +# get the IP address from network interface 1 +network.host: _gce:privateIp:1_ +# shortcut for _gce:privateIp:0_ +network.host: _gce_ +# Using GCE internal hostname (recommended) +network.host: _gce:hostname_ +-------------------------------------------------- + [[cloud-gce-usage-discovery-short]] ===== How to start (short story) diff --git a/docs/reference/modules/network.asciidoc b/docs/reference/modules/network.asciidoc index 70b4d8c97f9..2e48438b33a 100644 --- a/docs/reference/modules/network.asciidoc +++ b/docs/reference/modules/network.asciidoc @@ -54,6 +54,10 @@ provided network interface. For example `_en0:ipv6_`. When the `discovery-ec2` plugin is installed, you can use {plugins}/discovery-ec2-discovery.html#discovery-ec2-network-host[ec2 specific host settings]. +When the `cloud-gce` plugin is installed, you can use +{plugins}/discovery-gce-network-host.html[gce specific host settings]. + + [float] [[tcp-settings]] === TCP Settings diff --git a/plugins/cloud-gce/src/main/java/org/elasticsearch/cloud/gce/GceComputeService.java b/plugins/cloud-gce/src/main/java/org/elasticsearch/cloud/gce/GceComputeService.java index 6ba857db17a..ccb1eda3cc5 100644 --- a/plugins/cloud-gce/src/main/java/org/elasticsearch/cloud/gce/GceComputeService.java +++ b/plugins/cloud-gce/src/main/java/org/elasticsearch/cloud/gce/GceComputeService.java @@ -22,13 +22,14 @@ package org.elasticsearch.cloud.gce; import com.google.api.services.compute.model.Instance; import org.elasticsearch.common.component.LifecycleComponent; +import java.io.IOException; import java.util.Collection; /** * */ public interface GceComputeService extends LifecycleComponent { - static final public class Fields { + final class Fields { public static final String PROJECT = "cloud.gce.project_id"; public static final String ZONE = "cloud.gce.zone"; public static final String REFRESH = "cloud.gce.refresh_interval"; @@ -36,5 +37,24 @@ public interface GceComputeService extends LifecycleComponent public static final String VERSION = "Elasticsearch/GceCloud/1.0"; } - public Collection instances(); + /** + * Return a collection of running instances within the same GCE project + * @return a collection of running instances within the same GCE project + */ + Collection instances(); + + /** + *

Gets metadata on the current running machine (call to + * http://metadata.google.internal/computeMetadata/v1/instance/xxx).

+ *

For example, you can retrieve network information by replacing xxx with:

+ *
    + *
  • `hostname` when we need to resolve the host name
    • + *
  • `network-interfaces/0/ip` when we need to resolve private IP
    • + *
+ * @see org.elasticsearch.cloud.gce.network.GceNameResolver for bindings + * @param metadataPath path to metadata information + * @return extracted information (for example a hostname or an IP address) + * @throws IOException in case metadata URL is not accessible + */ + String metadata(String metadataPath) throws IOException; } diff --git a/plugins/cloud-gce/src/main/java/org/elasticsearch/cloud/gce/GceComputeServiceImpl.java b/plugins/cloud-gce/src/main/java/org/elasticsearch/cloud/gce/GceComputeServiceImpl.java index 2a9bf6d62d1..b205322c38c 100644 --- a/plugins/cloud-gce/src/main/java/org/elasticsearch/cloud/gce/GceComputeServiceImpl.java +++ b/plugins/cloud-gce/src/main/java/org/elasticsearch/cloud/gce/GceComputeServiceImpl.java @@ -21,6 +21,9 @@ package org.elasticsearch.cloud.gce; import com.google.api.client.googleapis.compute.ComputeCredential; import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; +import com.google.api.client.http.GenericUrl; +import com.google.api.client.http.HttpHeaders; +import com.google.api.client.http.HttpResponse; import com.google.api.client.http.HttpTransport; import com.google.api.client.json.JsonFactory; import com.google.api.client.json.jackson2.JacksonFactory; @@ -30,12 +33,15 @@ import com.google.api.services.compute.model.InstanceList; import org.elasticsearch.SpecialPermission; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.cloud.gce.network.GceNameResolver; import org.elasticsearch.common.component.AbstractLifecycleComponent; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.network.NetworkService; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import java.io.IOException; +import java.net.URL; import java.security.AccessController; import java.security.GeneralSecurityException; import java.security.PrivilegedActionException; @@ -54,7 +60,8 @@ public class GceComputeServiceImpl extends AbstractLifecycleComponent instances() { @@ -95,6 +102,37 @@ public class GceComputeServiceImpl extends AbstractLifecycleComponent() { + @Override + public HttpHeaders run() throws IOException { + return new HttpHeaders(); + } + }); + + // This is needed to query meta data: https://cloud.google.com/compute/docs/metadata + headers.put("Metadata-Flavor", "Google"); + HttpResponse response; + response = getGceHttpTransport().createRequestFactory() + .buildGetRequest(new GenericUrl(url)) + .setHeaders(headers) + .execute(); + String metadata = response.parseAsString(); + logger.debug("metadata found [{}]", metadata); + return metadata; + } catch (Exception e) { + throw new IOException("failed to fetch metadata from [" + urlMetadataNetwork + "]", e); + } + } + private Compute client; private TimeValue refreshInterval = null; private long lastRefresh; @@ -106,11 +144,12 @@ public class GceComputeServiceImpl extends AbstractLifecycleComponentResolves certain GCE related 'meta' hostnames into an actual hostname + * obtained from gce meta-data.

+ * Valid config values for {@link GceAddressResolverType}s are - + *
    + *
  • _gce_ - maps to privateIp
    • + *
  • _gce:privateIp_
    • + *
  • _gce:hostname_
    • + *
+ */ +public class GceNameResolver extends AbstractComponent implements CustomNameResolver { + + private final GceComputeService gceComputeService; + + /** + * enum that can be added to over time with more meta-data types + */ + private enum GceAddressResolverType { + + /** + * Using the hostname + */ + PRIVATE_DNS("gce:hostname", "hostname"), + /** + * Can be gce:privateIp, gce:privateIp:X where X is the network interface + */ + PRIVATE_IP("gce:privateIp", "network-interfaces/{{network}}/ip"), + /** + * same as "gce:privateIp" or "gce:privateIp:0" + */ + GCE("gce", PRIVATE_IP.gceName); + + final String configName; + final String gceName; + + GceAddressResolverType(String configName, String gceName) { + this.configName = configName; + this.gceName = gceName; + } + } + + /** + * Construct a {@link CustomNameResolver}. + */ + public GceNameResolver(Settings settings, GceComputeService gceComputeService) { + super(settings); + this.gceComputeService = gceComputeService; + } + + /** + * @param value the gce hostname type to discover. + * @return the appropriate host resolved from gce meta-data. + * @see CustomNameResolver#resolveIfPossible(String) + */ + private InetAddress[] resolve(String value) throws IOException { + String gceMetadataPath; + if (value.equals(GceAddressResolverType.GCE.configName)) { + // We replace network placeholder with default network interface value: 0 + gceMetadataPath = Strings.replace(GceAddressResolverType.GCE.gceName, "{{network}}", "0"); + } else if (value.equals(GceAddressResolverType.PRIVATE_DNS.configName)) { + gceMetadataPath = GceAddressResolverType.PRIVATE_DNS.gceName; + } else if (value.startsWith(GceAddressResolverType.PRIVATE_IP.configName)) { + // We extract the network interface from gce:privateIp:XX + String network = "0"; + String[] privateIpConfig = Strings.splitStringToArray(value, ':'); + if (privateIpConfig != null && privateIpConfig.length == 3) { + network = privateIpConfig[2]; + } + + // We replace network placeholder with network interface value + gceMetadataPath = Strings.replace(GceAddressResolverType.PRIVATE_IP.gceName, "{{network}}", network); + } else { + throw new IllegalArgumentException("[" + value + "] is not one of the supported GCE network.host setting. " + + "Expecting _gce_, _gce:privateIp:X_, _gce:hostname_"); + } + + try { + String metadataResult = gceComputeService.metadata(gceMetadataPath); + if (metadataResult == null || metadataResult.length() == 0) { + throw new IOException("no gce metadata returned from [" + gceMetadataPath + "] for [" + value + "]"); + } + // only one address: because we explicitly ask for only one via the GceHostnameType + return new InetAddress[] { InetAddress.getByName(metadataResult) }; + } catch (IOException e) { + throw new IOException("IOException caught when fetching InetAddress from [" + gceMetadataPath + "]", e); + } + } + + @Override + public InetAddress[] resolveDefault() { + return null; // using this, one has to explicitly specify _gce_ in network setting + } + + @Override + public InetAddress[] resolveIfPossible(String value) throws IOException { + // We only try to resolve network.host setting when it starts with _gce + if (value.startsWith("gce")) { + return resolve(value); + } + return null; + } +} diff --git a/plugins/cloud-gce/src/test/java/org/elasticsearch/discovery/gce/GceComputeServiceMock.java b/plugins/cloud-gce/src/test/java/org/elasticsearch/discovery/gce/GceComputeServiceMock.java index 1892297efe8..dcbd53fa95e 100644 --- a/plugins/cloud-gce/src/test/java/org/elasticsearch/discovery/gce/GceComputeServiceMock.java +++ b/plugins/cloud-gce/src/test/java/org/elasticsearch/discovery/gce/GceComputeServiceMock.java @@ -29,6 +29,7 @@ import com.google.api.client.testing.http.MockLowLevelHttpResponse; import org.elasticsearch.cloud.gce.GceComputeServiceImpl; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.Streams; +import org.elasticsearch.common.network.NetworkService; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.Callback; @@ -44,8 +45,8 @@ public class GceComputeServiceMock extends GceComputeServiceImpl { protected HttpTransport mockHttpTransport; - public GceComputeServiceMock(Settings settings) { - super(settings); + public GceComputeServiceMock(Settings settings, NetworkService networkService) { + super(settings, networkService); this.mockHttpTransport = configureMock(); } @@ -55,7 +56,7 @@ public class GceComputeServiceMock extends GceComputeServiceImpl { } protected HttpTransport configureMock() { - HttpTransport transport = new MockHttpTransport() { + return new MockHttpTransport() { @Override public LowLevelHttpRequest buildRequest(String method, final String url) throws IOException { return new MockLowLevelHttpRequest() { @@ -64,8 +65,8 @@ public class GceComputeServiceMock extends GceComputeServiceImpl { MockLowLevelHttpResponse response = new MockLowLevelHttpResponse(); response.setStatusCode(200); response.setContentType(Json.MEDIA_TYPE); - if (url.equals(TOKEN_SERVER_ENCODED_URL)) { - logger.info("--> Simulate GCE Auth response for [{}]", url); + if (url.startsWith(GCE_METADATA_URL)) { + logger.info("--> Simulate GCE Auth/Metadata response for [{}]", url); response.setContent(readGoogleInternalJsonResponse(url)); } else { logger.info("--> Simulate GCE API response for [{}]", url); @@ -77,8 +78,6 @@ public class GceComputeServiceMock extends GceComputeServiceImpl { }; } }; - - return transport; } private String readGoogleInternalJsonResponse(String url) throws IOException { @@ -91,23 +90,24 @@ public class GceComputeServiceMock extends GceComputeServiceImpl { private String readJsonResponse(String url, String urlRoot) throws IOException { // We extract from the url the mock file path we want to use - String mockFileName = Strings.replace(url, urlRoot, "") + ".json"; + String mockFileName = Strings.replace(url, urlRoot, ""); logger.debug("--> read mock file from [{}]", mockFileName); URL resource = GceComputeServiceMock.class.getResource(mockFileName); + if (resource == null) { + throw new IOException("can't read [" + url + "] in src/test/resources/org/elasticsearch/discovery/gce"); + } try (InputStream is = resource.openStream()) { final StringBuilder sb = new StringBuilder(); Streams.readAllLines(is, new Callback() { @Override public void handle(String s) { - sb.append(s).append("\n"); + sb.append(s); } }); String response = sb.toString(); logger.trace("{}", response); return response; - } catch (IOException e) { - throw e; } } } diff --git a/plugins/cloud-gce/src/test/java/org/elasticsearch/discovery/gce/GceDiscoveryTests.java b/plugins/cloud-gce/src/test/java/org/elasticsearch/discovery/gce/GceDiscoveryTests.java index b18cca1d84e..450ff72ca47 100644 --- a/plugins/cloud-gce/src/test/java/org/elasticsearch/discovery/gce/GceDiscoveryTests.java +++ b/plugins/cloud-gce/src/test/java/org/elasticsearch/discovery/gce/GceDiscoveryTests.java @@ -63,6 +63,7 @@ public class GceDiscoveryTests extends ESTestCase { protected static ThreadPool threadPool; protected MockTransportService transportService; + protected NetworkService networkService; protected GceComputeService mock; protected String projectName; @@ -91,6 +92,11 @@ public class GceDiscoveryTests extends ESTestCase { new LocalTransport(Settings.EMPTY, threadPool, Version.CURRENT, new NamedWriteableRegistry()), threadPool); } + @Before + public void createNetworkService() { + networkService = new NetworkService(Settings.EMPTY); + } + @After public void stopGceComputeService() { if (mock != null) { @@ -113,7 +119,7 @@ public class GceDiscoveryTests extends ESTestCase { .put(GceComputeService.Fields.PROJECT, projectName) .put(GceComputeService.Fields.ZONE, "europe-west1-b") .build(); - mock = new GceComputeServiceMock(nodeSettings); + mock = new GceComputeServiceMock(nodeSettings, networkService); List discoveryNodes = buildDynamicNodes(mock, nodeSettings); assertThat(discoveryNodes, hasSize(2)); } @@ -125,7 +131,7 @@ public class GceDiscoveryTests extends ESTestCase { .put(GceComputeService.Fields.ZONE, "europe-west1-b") .putArray(GceComputeService.Fields.TAGS, "elasticsearch") .build(); - mock = new GceComputeServiceMock(nodeSettings); + mock = new GceComputeServiceMock(nodeSettings, networkService); List discoveryNodes = buildDynamicNodes(mock, nodeSettings); assertThat(discoveryNodes, hasSize(1)); assertThat(discoveryNodes.get(0).getId(), is("#cloud-test2-0")); @@ -138,7 +144,7 @@ public class GceDiscoveryTests extends ESTestCase { .put(GceComputeService.Fields.ZONE, "europe-west1-b") .putArray(GceComputeService.Fields.TAGS, "elasticsearch", "dev") .build(); - mock = new GceComputeServiceMock(nodeSettings); + mock = new GceComputeServiceMock(nodeSettings, networkService); List discoveryNodes = buildDynamicNodes(mock, nodeSettings); assertThat(discoveryNodes, hasSize(1)); assertThat(discoveryNodes.get(0).getId(), is("#cloud-test2-0")); @@ -150,7 +156,7 @@ public class GceDiscoveryTests extends ESTestCase { .put(GceComputeService.Fields.PROJECT, projectName) .put(GceComputeService.Fields.ZONE, "europe-west1-b") .build(); - mock = new GceComputeServiceMock(nodeSettings); + mock = new GceComputeServiceMock(nodeSettings, networkService); List discoveryNodes = buildDynamicNodes(mock, nodeSettings); assertThat(discoveryNodes, hasSize(2)); } @@ -162,7 +168,7 @@ public class GceDiscoveryTests extends ESTestCase { .put(GceComputeService.Fields.ZONE, "europe-west1-b") .putArray(GceComputeService.Fields.TAGS, "elasticsearch") .build(); - mock = new GceComputeServiceMock(nodeSettings); + mock = new GceComputeServiceMock(nodeSettings, networkService); List discoveryNodes = buildDynamicNodes(mock, nodeSettings); assertThat(discoveryNodes, hasSize(2)); } @@ -174,7 +180,7 @@ public class GceDiscoveryTests extends ESTestCase { .put(GceComputeService.Fields.ZONE, "europe-west1-b") .putArray(GceComputeService.Fields.TAGS, "elasticsearch", "dev") .build(); - mock = new GceComputeServiceMock(nodeSettings); + mock = new GceComputeServiceMock(nodeSettings, networkService); List discoveryNodes = buildDynamicNodes(mock, nodeSettings); assertThat(discoveryNodes, hasSize(2)); } @@ -185,7 +191,7 @@ public class GceDiscoveryTests extends ESTestCase { .put(GceComputeService.Fields.PROJECT, projectName) .putArray(GceComputeService.Fields.ZONE, "us-central1-a", "europe-west1-b") .build(); - mock = new GceComputeServiceMock(nodeSettings); + mock = new GceComputeServiceMock(nodeSettings, networkService); List discoveryNodes = buildDynamicNodes(mock, nodeSettings); assertThat(discoveryNodes, hasSize(2)); } @@ -196,7 +202,7 @@ public class GceDiscoveryTests extends ESTestCase { .put(GceComputeService.Fields.PROJECT, projectName) .putArray(GceComputeService.Fields.ZONE, "us-central1-a", "europe-west1-b") .build(); - mock = new GceComputeServiceMock(nodeSettings); + mock = new GceComputeServiceMock(nodeSettings, networkService); List discoveryNodes = buildDynamicNodes(mock, nodeSettings); assertThat(discoveryNodes, hasSize(2)); } @@ -210,7 +216,7 @@ public class GceDiscoveryTests extends ESTestCase { .put(GceComputeService.Fields.PROJECT, projectName) .putArray(GceComputeService.Fields.ZONE, "us-central1-a", "us-central1-b") .build(); - mock = new GceComputeServiceMock(nodeSettings); + mock = new GceComputeServiceMock(nodeSettings, networkService); List discoveryNodes = buildDynamicNodes(mock, nodeSettings); assertThat(discoveryNodes, hasSize(0)); } diff --git a/plugins/cloud-gce/src/test/java/org/elasticsearch/discovery/gce/GceNetworkTests.java b/plugins/cloud-gce/src/test/java/org/elasticsearch/discovery/gce/GceNetworkTests.java new file mode 100644 index 00000000000..7550cdce7e4 --- /dev/null +++ b/plugins/cloud-gce/src/test/java/org/elasticsearch/discovery/gce/GceNetworkTests.java @@ -0,0 +1,132 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.discovery.gce; + +import org.elasticsearch.cloud.gce.network.GceNameResolver; +import org.elasticsearch.common.network.NetworkService; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.test.ESTestCase; +import org.junit.Test; + +import java.io.IOException; +import java.net.InetAddress; + +import static org.hamcrest.Matchers.arrayContaining; +import static org.hamcrest.Matchers.containsString; + +/** + * Test for GCE network.host settings. + * Related to https://github.com/elastic/elasticsearch/issues/13605 + */ +public class GceNetworkTests extends ESTestCase { + + /** + * Test for network.host: _gce_ + */ + @Test + public void networkHostGceDefault() throws IOException { + resolveGce("_gce_", InetAddress.getByName("10.240.0.2")); + } + + /** + * Test for network.host: _gce:privateIp_ + */ + @Test + public void networkHostPrivateIp() throws IOException { + resolveGce("_gce:privateIp_", InetAddress.getByName("10.240.0.2")); + } + + /** + * Test for network.host: _gce:hostname_ + */ + @Test + public void networkHostPrivateDns() throws IOException { + resolveGce("_gce:hostname_", InetAddress.getByName("localhost")); + } + + /** + * Test for network.host: _gce:doesnotexist_ + * This should raise an IllegalArgumentException as this setting does not exist + */ + @Test + public void networkHostWrongSetting() throws IOException { + resolveGce("_gce:doesnotexist_", (InetAddress) null); + } + + /** + * Test with multiple network interfaces: + * network.host: _gce:privateIp:0_ + * network.host: _gce:privateIp:1_ + */ + @Test + public void networkHostPrivateIpInterface() throws IOException { + resolveGce("_gce:privateIp:0_", InetAddress.getByName("10.240.0.2")); + resolveGce("_gce:privateIp:1_", InetAddress.getByName("10.150.0.1")); + } + + /** + * Test that we don't have any regression with network host core settings such as + * network.host: _local_ + */ + @Test + public void networkHostCoreLocal() throws IOException { + resolveGce("_local_", new NetworkService(Settings.EMPTY).resolveBindHostAddress(NetworkService.DEFAULT_NETWORK_HOST)); + } + + /** + * Utility test method to test different settings + * @param gceNetworkSetting tested network.host property + * @param expected expected InetAddress, null if we expect an exception + * @throws IOException Well... If something goes wrong :) + */ + private void resolveGce(String gceNetworkSetting, InetAddress expected) throws IOException { + resolveGce(gceNetworkSetting, expected == null ? null : new InetAddress [] { expected }); + } + + /** + * Utility test method to test different settings + * @param gceNetworkSetting tested network.host property + * @param expected expected InetAddress, null if we expect an exception + * @throws IOException Well... If something goes wrong :) + */ + private void resolveGce(String gceNetworkSetting, InetAddress[] expected) throws IOException { + Settings nodeSettings = Settings.builder() + .put("network.host", gceNetworkSetting) + .build(); + + NetworkService networkService = new NetworkService(nodeSettings); + GceComputeServiceMock mock = new GceComputeServiceMock(nodeSettings, networkService); + networkService.addCustomNameResolver(new GceNameResolver(nodeSettings, mock)); + try { + InetAddress[] addresses = networkService.resolveBindHostAddress(null); + if (expected == null) { + fail("We should get a IllegalArgumentException when setting network.host: _gce:doesnotexist_"); + } + assertThat(addresses, arrayContaining(expected)); + } catch (IllegalArgumentException e) { + if (expected != null) { + // We were expecting something and not an exception + throw e; + } + // We check that we get the expected exception + assertThat(e.getMessage(), containsString("is not one of the supported GCE network.host setting")); + } + } +} diff --git a/plugins/cloud-gce/src/test/resources/org/elasticsearch/discovery/gce/compute/v1/projects/multiplezonesandtwonodesindifferentzones/zones/europe-west1-b/instances.json b/plugins/cloud-gce/src/test/resources/org/elasticsearch/discovery/gce/compute/v1/projects/multiplezonesandtwonodesindifferentzones/zones/europe-west1-b/instances similarity index 100% rename from plugins/cloud-gce/src/test/resources/org/elasticsearch/discovery/gce/compute/v1/projects/multiplezonesandtwonodesindifferentzones/zones/europe-west1-b/instances.json rename to plugins/cloud-gce/src/test/resources/org/elasticsearch/discovery/gce/compute/v1/projects/multiplezonesandtwonodesindifferentzones/zones/europe-west1-b/instances diff --git a/plugins/cloud-gce/src/test/resources/org/elasticsearch/discovery/gce/compute/v1/projects/multiplezonesandtwonodesindifferentzones/zones/us-central1-a/instances.json b/plugins/cloud-gce/src/test/resources/org/elasticsearch/discovery/gce/compute/v1/projects/multiplezonesandtwonodesindifferentzones/zones/us-central1-a/instances similarity index 100% rename from plugins/cloud-gce/src/test/resources/org/elasticsearch/discovery/gce/compute/v1/projects/multiplezonesandtwonodesindifferentzones/zones/us-central1-a/instances.json rename to plugins/cloud-gce/src/test/resources/org/elasticsearch/discovery/gce/compute/v1/projects/multiplezonesandtwonodesindifferentzones/zones/us-central1-a/instances diff --git a/plugins/cloud-gce/src/test/resources/org/elasticsearch/discovery/gce/compute/v1/projects/multiplezonesandtwonodesinsamezone/zones/europe-west1-b/instances.json b/plugins/cloud-gce/src/test/resources/org/elasticsearch/discovery/gce/compute/v1/projects/multiplezonesandtwonodesinsamezone/zones/europe-west1-b/instances similarity index 100% rename from plugins/cloud-gce/src/test/resources/org/elasticsearch/discovery/gce/compute/v1/projects/multiplezonesandtwonodesinsamezone/zones/europe-west1-b/instances.json rename to plugins/cloud-gce/src/test/resources/org/elasticsearch/discovery/gce/compute/v1/projects/multiplezonesandtwonodesinsamezone/zones/europe-west1-b/instances diff --git a/plugins/cloud-gce/src/test/resources/org/elasticsearch/discovery/gce/compute/v1/projects/multiplezonesandtwonodesinsamezone/zones/us-central1-a/instances.json b/plugins/cloud-gce/src/test/resources/org/elasticsearch/discovery/gce/compute/v1/projects/multiplezonesandtwonodesinsamezone/zones/us-central1-a/instances similarity index 100% rename from plugins/cloud-gce/src/test/resources/org/elasticsearch/discovery/gce/compute/v1/projects/multiplezonesandtwonodesinsamezone/zones/us-central1-a/instances.json rename to plugins/cloud-gce/src/test/resources/org/elasticsearch/discovery/gce/compute/v1/projects/multiplezonesandtwonodesinsamezone/zones/us-central1-a/instances diff --git a/plugins/cloud-gce/src/test/resources/org/elasticsearch/discovery/gce/compute/v1/projects/nodeswithdifferenttagsandnotagset/zones/europe-west1-b/instances.json b/plugins/cloud-gce/src/test/resources/org/elasticsearch/discovery/gce/compute/v1/projects/nodeswithdifferenttagsandnotagset/zones/europe-west1-b/instances similarity index 100% rename from plugins/cloud-gce/src/test/resources/org/elasticsearch/discovery/gce/compute/v1/projects/nodeswithdifferenttagsandnotagset/zones/europe-west1-b/instances.json rename to plugins/cloud-gce/src/test/resources/org/elasticsearch/discovery/gce/compute/v1/projects/nodeswithdifferenttagsandnotagset/zones/europe-west1-b/instances diff --git a/plugins/cloud-gce/src/test/resources/org/elasticsearch/discovery/gce/compute/v1/projects/nodeswithdifferenttagsandonetagset/zones/europe-west1-b/instances.json b/plugins/cloud-gce/src/test/resources/org/elasticsearch/discovery/gce/compute/v1/projects/nodeswithdifferenttagsandonetagset/zones/europe-west1-b/instances similarity index 100% rename from plugins/cloud-gce/src/test/resources/org/elasticsearch/discovery/gce/compute/v1/projects/nodeswithdifferenttagsandonetagset/zones/europe-west1-b/instances.json rename to plugins/cloud-gce/src/test/resources/org/elasticsearch/discovery/gce/compute/v1/projects/nodeswithdifferenttagsandonetagset/zones/europe-west1-b/instances diff --git a/plugins/cloud-gce/src/test/resources/org/elasticsearch/discovery/gce/compute/v1/projects/nodeswithdifferenttagsandtwotagset/zones/europe-west1-b/instances.json b/plugins/cloud-gce/src/test/resources/org/elasticsearch/discovery/gce/compute/v1/projects/nodeswithdifferenttagsandtwotagset/zones/europe-west1-b/instances similarity index 100% rename from plugins/cloud-gce/src/test/resources/org/elasticsearch/discovery/gce/compute/v1/projects/nodeswithdifferenttagsandtwotagset/zones/europe-west1-b/instances.json rename to plugins/cloud-gce/src/test/resources/org/elasticsearch/discovery/gce/compute/v1/projects/nodeswithdifferenttagsandtwotagset/zones/europe-west1-b/instances diff --git a/plugins/cloud-gce/src/test/resources/org/elasticsearch/discovery/gce/compute/v1/projects/nodeswithsametagsandnotagset/zones/europe-west1-b/instances.json b/plugins/cloud-gce/src/test/resources/org/elasticsearch/discovery/gce/compute/v1/projects/nodeswithsametagsandnotagset/zones/europe-west1-b/instances similarity index 100% rename from plugins/cloud-gce/src/test/resources/org/elasticsearch/discovery/gce/compute/v1/projects/nodeswithsametagsandnotagset/zones/europe-west1-b/instances.json rename to plugins/cloud-gce/src/test/resources/org/elasticsearch/discovery/gce/compute/v1/projects/nodeswithsametagsandnotagset/zones/europe-west1-b/instances diff --git a/plugins/cloud-gce/src/test/resources/org/elasticsearch/discovery/gce/compute/v1/projects/nodeswithsametagsandonetagset/zones/europe-west1-b/instances.json b/plugins/cloud-gce/src/test/resources/org/elasticsearch/discovery/gce/compute/v1/projects/nodeswithsametagsandonetagset/zones/europe-west1-b/instances similarity index 100% rename from plugins/cloud-gce/src/test/resources/org/elasticsearch/discovery/gce/compute/v1/projects/nodeswithsametagsandonetagset/zones/europe-west1-b/instances.json rename to plugins/cloud-gce/src/test/resources/org/elasticsearch/discovery/gce/compute/v1/projects/nodeswithsametagsandonetagset/zones/europe-west1-b/instances diff --git a/plugins/cloud-gce/src/test/resources/org/elasticsearch/discovery/gce/compute/v1/projects/nodeswithsametagsandtwotagsset/zones/europe-west1-b/instances.json b/plugins/cloud-gce/src/test/resources/org/elasticsearch/discovery/gce/compute/v1/projects/nodeswithsametagsandtwotagsset/zones/europe-west1-b/instances similarity index 100% rename from plugins/cloud-gce/src/test/resources/org/elasticsearch/discovery/gce/compute/v1/projects/nodeswithsametagsandtwotagsset/zones/europe-west1-b/instances.json rename to plugins/cloud-gce/src/test/resources/org/elasticsearch/discovery/gce/compute/v1/projects/nodeswithsametagsandtwotagsset/zones/europe-west1-b/instances diff --git a/plugins/cloud-gce/src/test/resources/org/elasticsearch/discovery/gce/compute/v1/projects/zeronode43/zones/us-central1-a/instances.json b/plugins/cloud-gce/src/test/resources/org/elasticsearch/discovery/gce/compute/v1/projects/zeronode43/zones/us-central1-a/instances similarity index 100% rename from plugins/cloud-gce/src/test/resources/org/elasticsearch/discovery/gce/compute/v1/projects/zeronode43/zones/us-central1-a/instances.json rename to plugins/cloud-gce/src/test/resources/org/elasticsearch/discovery/gce/compute/v1/projects/zeronode43/zones/us-central1-a/instances diff --git a/plugins/cloud-gce/src/test/resources/org/elasticsearch/discovery/gce/compute/v1/projects/zeronode43/zones/us-central1-b/instances.json b/plugins/cloud-gce/src/test/resources/org/elasticsearch/discovery/gce/compute/v1/projects/zeronode43/zones/us-central1-b/instances similarity index 100% rename from plugins/cloud-gce/src/test/resources/org/elasticsearch/discovery/gce/compute/v1/projects/zeronode43/zones/us-central1-b/instances.json rename to plugins/cloud-gce/src/test/resources/org/elasticsearch/discovery/gce/compute/v1/projects/zeronode43/zones/us-central1-b/instances diff --git a/plugins/cloud-gce/src/test/resources/org/elasticsearch/discovery/gce/computeMetadata/v1/instance/hostname b/plugins/cloud-gce/src/test/resources/org/elasticsearch/discovery/gce/computeMetadata/v1/instance/hostname new file mode 100644 index 00000000000..2fbb50c4a8d --- /dev/null +++ b/plugins/cloud-gce/src/test/resources/org/elasticsearch/discovery/gce/computeMetadata/v1/instance/hostname @@ -0,0 +1 @@ +localhost diff --git a/plugins/cloud-gce/src/test/resources/org/elasticsearch/discovery/gce/computeMetadata/v1/instance/network-interfaces/0/ip b/plugins/cloud-gce/src/test/resources/org/elasticsearch/discovery/gce/computeMetadata/v1/instance/network-interfaces/0/ip new file mode 100644 index 00000000000..1ac79d6027d --- /dev/null +++ b/plugins/cloud-gce/src/test/resources/org/elasticsearch/discovery/gce/computeMetadata/v1/instance/network-interfaces/0/ip @@ -0,0 +1 @@ +10.240.0.2 diff --git a/plugins/cloud-gce/src/test/resources/org/elasticsearch/discovery/gce/computeMetadata/v1/instance/network-interfaces/1/ip b/plugins/cloud-gce/src/test/resources/org/elasticsearch/discovery/gce/computeMetadata/v1/instance/network-interfaces/1/ip new file mode 100644 index 00000000000..e3bb0f875b6 --- /dev/null +++ b/plugins/cloud-gce/src/test/resources/org/elasticsearch/discovery/gce/computeMetadata/v1/instance/network-interfaces/1/ip @@ -0,0 +1 @@ +10.150.0.1 diff --git a/plugins/cloud-gce/src/test/resources/org/elasticsearch/discovery/gce/computeMetadata/v1/instance/service-accounts/default/token.json b/plugins/cloud-gce/src/test/resources/org/elasticsearch/discovery/gce/computeMetadata/v1/instance/service-accounts/default/token similarity index 100% rename from plugins/cloud-gce/src/test/resources/org/elasticsearch/discovery/gce/computeMetadata/v1/instance/service-accounts/default/token.json rename to plugins/cloud-gce/src/test/resources/org/elasticsearch/discovery/gce/computeMetadata/v1/instance/service-accounts/default/token diff --git a/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/network/Ec2NameResolver.java b/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/network/Ec2NameResolver.java index f04d3ecc937..3788f8237dc 100755 --- a/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/network/Ec2NameResolver.java +++ b/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/network/Ec2NameResolver.java @@ -91,31 +91,25 @@ public class Ec2NameResolver extends AbstractComponent implements CustomNameReso * @return the appropriate host resolved from ec2 meta-data, or null if it cannot be obtained. * @see CustomNameResolver#resolveIfPossible(String) */ - public InetAddress[] resolve(Ec2HostnameType type, boolean warnOnFailure) { - URLConnection urlConnection = null; + public InetAddress[] resolve(Ec2HostnameType type) throws IOException { InputStream in = null; + String metadataUrl = AwsEc2ServiceImpl.EC2_METADATA_URL + type.ec2Name; try { - URL url = new URL(AwsEc2ServiceImpl.EC2_METADATA_URL + type.ec2Name); + URL url = new URL(metadataUrl); logger.debug("obtaining ec2 hostname from ec2 meta-data url {}", url); - urlConnection = url.openConnection(); + URLConnection urlConnection = url.openConnection(); urlConnection.setConnectTimeout(2000); in = urlConnection.getInputStream(); BufferedReader urlReader = new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8)); String metadataResult = urlReader.readLine(); if (metadataResult == null || metadataResult.length() == 0) { - logger.error("no ec2 metadata returned from {}", url); - return null; + throw new IOException("no gce metadata returned from [" + url + "] for [" + type.configName + "]"); } // only one address: because we explicitly ask for only one via the Ec2HostnameType return new InetAddress[] { InetAddress.getByName(metadataResult) }; } catch (IOException e) { - if (warnOnFailure) { - logger.warn("failed to get metadata for [" + type.configName + "]", e); - } else { - logger.debug("failed to get metadata for [" + type.configName + "]", e); - } - return null; + throw new IOException("IOException caught when fetching InetAddress from [" + metadataUrl + "]", e); } finally { IOUtils.closeWhileHandlingException(in); } @@ -128,10 +122,10 @@ public class Ec2NameResolver extends AbstractComponent implements CustomNameReso } @Override - public InetAddress[] resolveIfPossible(String value) { + public InetAddress[] resolveIfPossible(String value) throws IOException { for (Ec2HostnameType type : Ec2HostnameType.values()) { if (type.configName.equals(value)) { - return resolve(type, true); + return resolve(type); } } return null; diff --git a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2NetworkTests.java b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2NetworkTests.java new file mode 100644 index 00000000000..8aa9ca56a3f --- /dev/null +++ b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2NetworkTests.java @@ -0,0 +1,187 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.discovery.ec2; + +import org.elasticsearch.cloud.aws.network.Ec2NameResolver; +import org.elasticsearch.common.network.NetworkService; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.test.ESTestCase; +import org.junit.Test; + +import java.io.IOException; +import java.net.InetAddress; + +import static org.hamcrest.Matchers.arrayContaining; +import static org.hamcrest.Matchers.containsString; + +/** + * Test for EC2 network.host settings. + */ +public class Ec2NetworkTests extends ESTestCase { + + /** + * Test for network.host: _ec2_ + */ + @Test + public void networkHostEc2() throws IOException { + Settings nodeSettings = Settings.builder() + .put("network.host", "_ec2_") + .build(); + + NetworkService networkService = new NetworkService(nodeSettings); + networkService.addCustomNameResolver(new Ec2NameResolver(nodeSettings)); + // TODO we need to replace that with a mock. For now we check the URL we are supposed to reach. + try { + networkService.resolveBindHostAddress(null); + } catch (IOException e) { + assertThat(e.getMessage(), containsString("local-ipv4")); + } + } + + /** + * Test for network.host: _ec2:publicIp_ + */ + @Test + public void networkHostEc2PublicIp() throws IOException { + Settings nodeSettings = Settings.builder() + .put("network.host", "_ec2:publicIp_") + .build(); + + NetworkService networkService = new NetworkService(nodeSettings); + networkService.addCustomNameResolver(new Ec2NameResolver(nodeSettings)); + // TODO we need to replace that with a mock. For now we check the URL we are supposed to reach. + try { + networkService.resolveBindHostAddress(null); + } catch (IOException e) { + assertThat(e.getMessage(), containsString("public-ipv4")); + } + } + + /** + * Test for network.host: _ec2:privateIp_ + */ + @Test + public void networkHostEc2PrivateIp() throws IOException { + Settings nodeSettings = Settings.builder() + .put("network.host", "_ec2:privateIp_") + .build(); + + NetworkService networkService = new NetworkService(nodeSettings); + networkService.addCustomNameResolver(new Ec2NameResolver(nodeSettings)); + // TODO we need to replace that with a mock. For now we check the URL we are supposed to reach. + try { + networkService.resolveBindHostAddress(null); + } catch (IOException e) { + assertThat(e.getMessage(), containsString("local-ipv4")); + } + } + + /** + * Test for network.host: _ec2:privateIpv4_ + */ + @Test + public void networkHostEc2PrivateIpv4() throws IOException { + Settings nodeSettings = Settings.builder() + .put("network.host", "_ec2:privateIpv4_") + .build(); + + NetworkService networkService = new NetworkService(nodeSettings); + networkService.addCustomNameResolver(new Ec2NameResolver(nodeSettings)); + // TODO we need to replace that with a mock. For now we check the URL we are supposed to reach. + try { + networkService.resolveBindHostAddress(null); + } catch (IOException e) { + assertThat(e.getMessage(), containsString("local-ipv4")); + } + } + + /** + * Test for network.host: _ec2:privateDns_ + */ + @Test + public void networkHostEc2PrivateDns() throws IOException { + Settings nodeSettings = Settings.builder() + .put("network.host", "_ec2:privateDns_") + .build(); + + NetworkService networkService = new NetworkService(nodeSettings); + networkService.addCustomNameResolver(new Ec2NameResolver(nodeSettings)); + // TODO we need to replace that with a mock. For now we check the URL we are supposed to reach. + try { + networkService.resolveBindHostAddress(null); + } catch (IOException e) { + assertThat(e.getMessage(), containsString("local-hostname")); + } + } + + /** + * Test for network.host: _ec2:publicIpv4_ + */ + @Test + public void networkHostEc2PublicIpv4() throws IOException { + Settings nodeSettings = Settings.builder() + .put("network.host", "_ec2:publicIpv4_") + .build(); + + NetworkService networkService = new NetworkService(nodeSettings); + networkService.addCustomNameResolver(new Ec2NameResolver(nodeSettings)); + // TODO we need to replace that with a mock. For now we check the URL we are supposed to reach. + try { + networkService.resolveBindHostAddress(null); + } catch (IOException e) { + assertThat(e.getMessage(), containsString("public-ipv4")); + } + } + + /** + * Test for network.host: _ec2:publicDns_ + */ + @Test + public void networkHostEc2PublicDns() throws IOException { + Settings nodeSettings = Settings.builder() + .put("network.host", "_ec2:publicDns_") + .build(); + + NetworkService networkService = new NetworkService(nodeSettings); + networkService.addCustomNameResolver(new Ec2NameResolver(nodeSettings)); + // TODO we need to replace that with a mock. For now we check the URL we are supposed to reach. + try { + networkService.resolveBindHostAddress(null); + } catch (IOException e) { + assertThat(e.getMessage(), containsString("public-hostname")); + } + } + + /** + * Test that we don't have any regression with network host core settings such as + * network.host: _local_ + */ + @Test + public void networkHostCoreLocal() throws IOException { + Settings nodeSettings = Settings.builder() + .put("network.host", "_local_") + .build(); + + NetworkService networkService = new NetworkService(nodeSettings); + networkService.addCustomNameResolver(new Ec2NameResolver(nodeSettings)); + InetAddress[] addresses = networkService.resolveBindHostAddress(null); + assertThat(addresses, arrayContaining(networkService.resolveBindHostAddress("_local_"))); + } +}