honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'pr/22566-ec2-meta-host'

This commit is contained in:
David Pilato 2017-01-26 17:37:53 +01:00
parent a475323aa1 6e7aee0c5a
commit 59b34ba71c
4 changed files with 94 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
docs/plugins/discovery-ec2.asciidoc
View File

@ -172,9 +172,20 @@ The following are a list of settings (prefixed with `discovery.ec2`) that can fu
`host_type`::
The type of host type to use to communicate with other instances. Can be
one of `private_ip`, `public_ip`, `private_dns`, `public_dns`. Defaults to
`private_ip`.
+
--
The type of host type to use to communicate with other instances. Can be
one of `private_ip`, `public_ip`, `private_dns`, `public_dns` or `tag:TAGNAME` where
`TAGNAME` refers to a name of a tag configured for all EC2 instances. Instances which don't
have this tag set will be ignored by the discovery process.
For example if you defined a tag `my-elasticsearch-host` in ec2 and set it to `myhostname1.mydomain.com`, then
setting `host_type: tag:my-elasticsearch-host` will tell Discovery Ec2 plugin to read the host name from the
`my-elasticsearch-host` tag. In this case, it will be resolved to `myhostname1.mydomain.com`.
http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html[Read more about EC2 Tags].
Defaults to `private_ip`.
--
`availability_zones`::

20
plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/AwsEc2Service.java
View File

@ -164,20 +164,22 @@ public interface AwsEc2Service {
* Defines discovery settings for ec2. Starting with discovery.ec2.
*/
interface DISCOVERY_EC2 {
enum HostType {
PRIVATE_IP,
PUBLIC_IP,
PRIVATE_DNS,
PUBLIC_DNS
class HostType {
public static final String PRIVATE_IP = "private_ip";
public static final String PUBLIC_IP = "public_ip";
public static final String PRIVATE_DNS = "private_dns";
public static final String PUBLIC_DNS = "public_dns";
public static final String TAG_PREFIX = "tag:";
}
/**
* discovery.ec2.host_type: The type of host type to use to communicate with other instances.
* Can be one of private_ip, public_ip, private_dns, public_dns. Defaults to private_ip.
* Can be one of private_ip, public_ip, private_dns, public_dns or tag:XXXX where
* XXXX refers to a name of a tag configured for all EC2 instances. Instances which don't
* have this tag set will be ignored by the discovery process. Defaults to private_ip.
*/
Setting<HostType> HOST_TYPE_SETTING =
new Setting<>("discovery.ec2.host_type", HostType.PRIVATE_IP.name(), s -> HostType.valueOf(s.toUpperCase(Locale.ROOT)),
Property.NodeScope);
Setting<String> HOST_TYPE_SETTING =
new Setting<>("discovery.ec2.host_type", HostType.PRIVATE_IP, Function.identity(), Property.NodeScope);
/**
* discovery.ec2.any_group: If set to false, will require all security groups to be present for the instance to be used for the
* discovery. Defaults to true.

46
plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2UnicastHostsProvider.java
View File

@ -27,6 +27,7 @@ import com.amazonaws.services.ec2.model.Filter;
import com.amazonaws.services.ec2.model.GroupIdentifier;
import com.amazonaws.services.ec2.model.Instance;
import com.amazonaws.services.ec2.model.Reservation;
import com.amazonaws.services.ec2.model.Tag;
import org.apache.logging.log4j.message.ParameterizedMessage;
import org.apache.logging.log4j.util.Supplier;
import org.elasticsearch.Version;
@ -51,6 +52,11 @@ import java.util.Set;
import static java.util.Collections.disjoint;
import static java.util.Collections.emptyMap;
import static java.util.Collections.emptySet;
import static org.elasticsearch.cloud.aws.AwsEc2Service.DISCOVERY_EC2.HostType.TAG_PREFIX;
import static org.elasticsearch.cloud.aws.AwsEc2Service.DISCOVERY_EC2.HostType.PRIVATE_DNS;
import static org.elasticsearch.cloud.aws.AwsEc2Service.DISCOVERY_EC2.HostType.PRIVATE_IP;
import static org.elasticsearch.cloud.aws.AwsEc2Service.DISCOVERY_EC2.HostType.PUBLIC_DNS;
import static org.elasticsearch.cloud.aws.AwsEc2Service.DISCOVERY_EC2.HostType.PUBLIC_IP;
public class AwsEc2UnicastHostsProvider extends AbstractComponent implements UnicastHostsProvider {
@ -66,7 +72,7 @@ public class AwsEc2UnicastHostsProvider extends AbstractComponent implements Uni
private final Set<String> availabilityZones;
private final DISCOVERY_EC2.HostType hostType;
private final String hostType;
private final DiscoNodesCache discoNodes;
@ -149,19 +155,27 @@ public class AwsEc2UnicastHostsProvider extends AbstractComponent implements Uni
}
String address = null;
switch (hostType) {
case PRIVATE_DNS:
address = instance.getPrivateDnsName();
break;
case PRIVATE_IP:
address = instance.getPrivateIpAddress();
break;
case PUBLIC_DNS:
address = instance.getPublicDnsName();
break;
case PUBLIC_IP:
address = instance.getPublicIpAddress();
break;
if (hostType.equals(PRIVATE_DNS)) {
address = instance.getPrivateDnsName();
} else if (hostType.equals(PRIVATE_IP)) {
address = instance.getPrivateIpAddress();
} else if (hostType.equals(PUBLIC_DNS)) {
address = instance.getPublicDnsName();
} else if (hostType.equals(PUBLIC_IP)) {
address = instance.getPublicIpAddress();
} else if (hostType.startsWith(TAG_PREFIX)) {
// Reading the node host from its metadata
String tagName = hostType.substring(TAG_PREFIX.length());
logger.debug("reading hostname from [{}] instance tag", tagName);
List<Tag> tags = instance.getTags();
for (Tag tag : tags) {
if (tag.getKey().equals(tagName)) {
address = tag.getValue();
logger.debug("using [{}] as the instance address", address);
}
}
} else {
throw new IllegalArgumentException(hostType + " is unknown for discovery.ec2.host_type");
}
if (address != null) {
try {
@ -169,8 +183,8 @@ public class AwsEc2UnicastHostsProvider extends AbstractComponent implements Uni
TransportAddress[] addresses = transportService.addressesFromString(address, 1);
for (int i = 0; i < addresses.length; i++) {
logger.trace("adding {}, address {}, transport_address {}", instance.getInstanceId(), address, addresses[i]);
discoNodes.add(new DiscoveryNode("#cloud-" + instance.getInstanceId() + "-" + i, addresses[i],
emptyMap(), emptySet(), Version.CURRENT.minimumCompatibilityVersion()));
discoNodes.add(new DiscoveryNode(instance.getInstanceId(), "#cloud-" + instance.getInstanceId() + "-" + i,
addresses[i], emptyMap(), emptySet(), Version.CURRENT.minimumCompatibilityVersion()));
}
} catch (Exception e) {
final String finalAddress = address;

44
plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryTests.java
View File

@ -41,6 +41,7 @@ import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.Collections;
@ -51,6 +52,7 @@ import java.util.concurrent.ConcurrentHashMap;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.hasSize;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.isOneOf;
public class Ec2DiscoveryTests extends ESTestCase {
@ -195,12 +197,11 @@ public class Ec2DiscoveryTests extends ESTestCase {
Settings nodeSettings = Settings.builder()
.put(DISCOVERY_EC2.HOST_TYPE_SETTING.getKey(), "does_not_exist")
.build();
try {
IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> {
buildDynamicNodes(nodeSettings, 1);
fail("Expected IllegalArgumentException");
} catch (IllegalArgumentException e) {
assertThat(e.getMessage(), containsString("No enum constant"));
}
});
assertThat(exception.getMessage(), containsString("does_not_exist is unknown for discovery.ec2.host_type"));
}
public void testFilterByTags() throws InterruptedException {
@ -259,6 +260,39 @@ public class Ec2DiscoveryTests extends ESTestCase {
assertThat(discoveryNodes, hasSize(prodInstances));
}
public void testReadHostFromTag() throws InterruptedException, UnknownHostException {
int nodes = randomIntBetween(5, 10);
String[] addresses = new String[nodes];
for (int node = 0; node < nodes; node++) {
addresses[node] = "192.168.0." + (node + 1);
poorMansDNS.put("node" + (node + 1), new TransportAddress(InetAddress.getByName(addresses[node]), 9300));
}
Settings nodeSettings = Settings.builder()
.put(DISCOVERY_EC2.HOST_TYPE_SETTING.getKey(), "tag:foo")
.build();
List<List<Tag>> tagsList = new ArrayList<>();
for (int node = 0; node < nodes; node++) {
List<Tag> tags = new ArrayList<>();
tags.add(new Tag("foo", "node" + (node + 1)));
tagsList.add(tags);
}
logger.info("started [{}] instances", nodes);
List<DiscoveryNode> discoveryNodes = buildDynamicNodes(nodeSettings, nodes, tagsList);
assertThat(discoveryNodes, hasSize(nodes));
for (DiscoveryNode discoveryNode : discoveryNodes) {
TransportAddress address = discoveryNode.getAddress();
TransportAddress expected = poorMansDNS.get(discoveryNode.getName());
assertEquals(address, expected);
}
}
abstract class DummyEc2HostProvider extends AwsEc2UnicastHostsProvider {
public int fetchCount = 0;
public DummyEc2HostProvider(Settings settings, TransportService transportService, AwsEc2Service service) {