Apache
/
jclouds
mirror of https://github.com/apache/jclouds.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Issue 452: renamed description -> name 

made nodes a list instead of a map

added credential_url
This commit is contained in:
Adrian Cole 2011-01-28 15:47:24 -08:00
parent d14dca16e7
commit ce82c05222
14 changed files with 210 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

76
sandbox-apis/byon/README.txt Normal file
View File

@ -0,0 +1,76 @@
= Bring Your Own Nodes to the jclouds ComputeService =
The bring your own node provider (byon) allows you to specify a source which jclouds will read
nodes from. Using this, you can have jclouds control your standalone machines, or even cloud
hosts that are sitting idle.
== Constraints ==
The byon provider only supports the following functions of ComputeService:
* listNodes
* listNodesDetailsMatching
* getNodeMetadata
* runScriptOnNodesMatching
== How to use the byon provider ==
The byon provider requires you supply a list of nodes using a property. Here are
the valid properties you can use:
* byon.endpoint - url to access the list, can be http://, file://, classpath://
* byon.nodes - inline defined yaml in string form.
Note:
The identity and credential fields of the ComputeServiceContextFactory are ignored.
=== Java example ===
Properties props = new Properties();
// if you built the yaml string by hand
props.setProperty("byon.nodes", stringLiteral);
// or you can specify an external reference
props.setProperty("byon.endpoint", "file://path/to/byon.yaml");
// or you can specify a file in your classpath
props.setProperty("byon.endpoint", "classpath:///byon.yaml");
context = new ComputeServiceContextFactory().createContext("byon", "foo", "bar",
ImmutableSet.<Module> of(new JschSshClientModule()), props);
== File format ==
You must define your nodes in yaml, and they must be in a collection called nodes.
Here are the properties:
* id - opaque unique id
* name - optional; user specified name
* hostname - name or ip address to contact the node on
* os_arch - ex. x86
* os_family - must conform to org.jclouds.compute.domain.OsFamily in lower-hyphen format
ex. rhel, ubuntu, centos, debian, amzn-linux
* os_name - ex. redhat
* os_version - normalized to numbers when possible. ex. for centos: 5.3, ubuntu: 10.10
* group - primary group of the machine. ex. hadoop
* tags - list of arbitrary tags. * note this list is not yet in jclouds NodeMetadata
* username - primary login user to the os. ex. ubuntu, vcloud, root
* sudo_password - optional; base 64 encoded sudo password (ex. input to sudo -S)
one of:
* credential - base 64 encoded RSA private key or password
* credential_url - location of plain-text RSA private key or password.
ex. file:///home/me/.ssh/id_rsa
classpath:///id_rsa
=== Example File ===
nodes:
- id: cluster-1
name: cluster-1
hostname: cluster-1.mydomain.com
os_arch: x86
os_family: rhel
os_name: redhat
os_version: 5.3
group: hadoop
tags:
- vanilla
username: myUser
credential: ZmFuY3lmb290
sudo_password: c3Vkbw==

23
sandbox-apis/byon/src/main/java/org/jclouds/byon/Node.java
View File

@ -19,6 +19,7 @@
package org.jclouds.byon;
import java.net.URI;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
@ -35,11 +36,11 @@ public class Node {
public Node() {
}
public Node(String id, String name, String description, String hostname, String osArch, String osFamily, String osName,
String osVersion, String group, List<String> tags, String username, String credential, String sudo_password) {
public Node(String id, String name, String hostname, String osArch, String osFamily,
String osName, String osVersion, String group, List<String> tags, String username, String credential,
URI credentialUrl, String sudo_password) {
this.id = id;
this.name = name;
this.description = description;
this.hostname = hostname;
this.os_arch = osArch;
this.os_family = osFamily;
@ -49,13 +50,13 @@ public class Node {
this.tags = ImmutableList.copyOf(tags);
this.username = username;
this.credential = credential;
this.credential_url = credentialUrl != null ? credentialUrl.toASCIIString() : null;
this.sudo_password = sudo_password;
}
// public due to snakeyaml
public String id;
public String name;
public String description;
public String hostname;
public String os_arch;
public String os_family;
@ -65,6 +66,7 @@ public class Node {
public List<String> tags;
public String username;
public String credential;
public String credential_url;
public String sudo_password;
public String getId() {
@ -79,10 +81,6 @@ public class Node {
return group;
}
public String getDescription() {
return description;
}
public String getHostname() {
return hostname;
}
@ -118,6 +116,10 @@ public class Node {
return credential;
}
public URI getCredentialUrl() {
return credential_url != null ? URI.create(credential_url) : null;
}
@Override
public int hashCode() {
return Objects.hashCode(id);
@ -136,10 +138,11 @@ public class Node {
@Override
public String toString() {
return Objects.toStringHelper(this).add("id", id).add("name", name).add("description", description).add(
return Objects.toStringHelper(this).add("id", id).add("name", name).add(
"hostname", hostname).add("osArch", os_arch).add("osFamily", os_family).add("osName", os_name).add(
"osVersion", os_version).add("group", group).add("tags", tags).add("username", username).add(
"hasCredential", credential != null).add("hasSudoPassword", sudo_password != null).toString();
"hasCredential", credential != null || credential_url != null).add("hasSudoPassword",
sudo_password != null).toString();
}
}

3
sandbox-apis/byon/src/main/java/org/jclouds/byon/config/BYONComputeServiceContextModule.java
View File

@ -20,6 +20,7 @@
package org.jclouds.byon.config;
import java.io.InputStream;
import java.net.URI;
import java.util.Map;
import javax.inject.Singleton;
@ -65,6 +66,8 @@ public class BYONComputeServiceContextModule extends
}).to(NodesParsedFromSupplier.class);
bind(new TypeLiteral<Supplier<InputStream>>() {
}).annotatedWith(Provider.class).to(SupplyFromProviderURIOrNodesProperty.class);
bind(new TypeLiteral<Function<URI, InputStream>>() {
}).to(SupplyFromProviderURIOrNodesProperty.class);
// TODO make this somehow overridable via user request
bind(new TypeLiteral<Function<InputStream, Map<String, Node>>>() {
}).to(NodesFromYaml.class);

42
sandbox-apis/byon/src/main/java/org/jclouds/byon/functions/NodeToNodeMetadata.java
View File

@ -21,8 +21,12 @@ package org.jclouds.byon.functions;
import static com.google.common.base.Preconditions.checkNotNull;
import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
import java.util.Map;
import javax.annotation.Resource;
import javax.inject.Inject;
import javax.inject.Singleton;
@ -35,6 +39,8 @@ import org.jclouds.compute.domain.OsFamily;
import org.jclouds.crypto.CryptoStreams;
import org.jclouds.domain.Credentials;
import org.jclouds.domain.Location;
import org.jclouds.logging.Logger;
import org.jclouds.util.Strings2;
import com.google.common.base.Charsets;
import com.google.common.base.Function;
@ -46,13 +52,19 @@ import com.google.common.collect.ImmutableSet;
*/
@Singleton
public class NodeToNodeMetadata implements Function<Node, NodeMetadata> {
@Resource
protected Logger logger = Logger.NULL;
private final Supplier<Location> location;
private final Map<String, Credentials> credentialStore;
private final Function<URI, InputStream> slurp;
@Inject
NodeToNodeMetadata(Supplier<Location> location, Map<String, Credentials> credentialStore) {
NodeToNodeMetadata(Supplier<Location> location, Function<URI, InputStream> slurp,
Map<String, Credentials> credentialStore) {
this.location = checkNotNull(location, "location");
this.credentialStore = checkNotNull(credentialStore, "credentialStore");
this.slurp = checkNotNull(slurp, "slurp");
}
@Override
@ -64,17 +76,31 @@ public class NodeToNodeMetadata implements Function<Node, NodeMetadata> {
builder.tag(from.getGroup());
// TODO add tags!
builder.operatingSystem(new OperatingSystemBuilder().arch(from.getOsArch()).family(
OsFamily.fromValue(from.getOsFamily())).name(from.getOsName()).version(from.getOsVersion()).description(
from.getDescription()).build());
OsFamily.fromValue(from.getOsFamily())).name(from.getOsName()).description(from.getOsName()).version(
from.getOsVersion()).build());
builder.state(NodeState.RUNNING);
builder.publicAddresses(ImmutableSet.<String> of(from.getHostname()));
Credentials creds = new Credentials(from.getUsername(), new String(CryptoStreams.base64(from.getCredential()),
Charsets.UTF_8));
builder.credentials(creds);
if (from.getUsername() != null) {
Credentials creds = null;
if (from.getCredentialUrl() != null) {
try {
creds = new Credentials(from.getUsername(), Strings2.toStringAndClose(slurp.apply(from
.getCredentialUrl())));
} catch (IOException e) {
logger.error(e, "URI could not be read: %s", from.getCredentialUrl());
}
} else if (from.getCredential() != null) {
creds = new Credentials(from.getUsername(), new String(CryptoStreams.base64(from.getCredential()),
Charsets.UTF_8));
}
if (creds != null)
builder.credentials(creds);
credentialStore.put("node#" + from.getId(), creds);
}
if (from.getSudoPassword() != null)
builder.adminPassword(new String(CryptoStreams.base64(from.getSudoPassword()), Charsets.UTF_8));
credentialStore.put("node#" + from.getId(), creds);
return builder.build();
}
}

8
sandbox-apis/byon/src/main/java/org/jclouds/byon/functions/NodesFromYaml.java
View File

@ -54,6 +54,8 @@ import com.google.common.collect.Maps;
* - vanilla
* username: kelvin
* credential: password_or_rsa_in_base64
* or
* credential_url: password_or_rsa_file ex. resource:///id_rsa will get the classpath /id_rsa.pub; file://path/to/id_rsa
* sudo_password: password_in_base64
* </pre>
*
@ -87,8 +89,10 @@ public class NodesFromYaml implements Function<InputStream, Map<String, Node>> {
checkState(config != null, "missing config: class");
checkState(config.nodes != null, "missing nodes: collection");
return Maps.uniqueIndex(config.nodes, new Function<Node, String>(){
public String apply(Node node) { return node.getId(); }
return Maps.uniqueIndex(config.nodes, new Function<Node, String>() {
public String apply(Node node) {
return node.getId();
}
});
}
}

26
sandbox-apis/byon/src/main/java/org/jclouds/byon/suppliers/SupplyFromProviderURIOrNodesProperty.java
View File

@ -32,6 +32,7 @@ import org.jclouds.logging.Logger;
import org.jclouds.util.Strings2;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Function;
import com.google.common.base.Supplier;
import com.google.common.base.Throwables;
import com.google.inject.Inject;
@ -40,7 +41,7 @@ import com.google.inject.name.Named;
/**
* @author Adrian Cole
*/
public class SupplyFromProviderURIOrNodesProperty implements Supplier<InputStream> {
public class SupplyFromProviderURIOrNodesProperty implements Supplier<InputStream>, Function<URI, InputStream> {
@Resource
protected Logger logger = Logger.NULL;
private final URI url;
@ -51,7 +52,7 @@ public class SupplyFromProviderURIOrNodesProperty implements Supplier<InputStrea
String nodes;
@Inject
SupplyFromProviderURIOrNodesProperty(@Provider URI url) {
public SupplyFromProviderURIOrNodesProperty(@Provider URI url) {
this.url = checkNotNull(url, "url");
}
@ -59,13 +60,7 @@ public class SupplyFromProviderURIOrNodesProperty implements Supplier<InputStrea
public InputStream get() {
if (nodes != null)
return Strings2.toInputStream(nodes);
try {
return url.toURL().openStream();
} catch (IOException e) {
logger.error(e, "URI could not be read: %s", url);
Throwables.propagate(e);
return null;
}
return apply(url);
}
@Override
@ -73,4 +68,17 @@ public class SupplyFromProviderURIOrNodesProperty implements Supplier<InputStrea
return "[url=" + url + "]";
}
@Override
public InputStream apply(URI input) {
try {
if (input.getScheme() != null && input.getScheme().equals("classpath"))
return getClass().getResourceAsStream(input.getPath());
return input.toURL().openStream();
} catch (IOException e) {
logger.error(e, "URI could not be read: %s", url);
Throwables.propagate(e);
return null;
}
}
}

7
sandbox-apis/byon/src/test/java/org/jclouds/byon/BYONComputeServiceLiveTest.java
View File

@ -20,7 +20,6 @@
package org.jclouds.byon;
import static org.jclouds.compute.options.RunScriptOptions.Builder.wrapInInitScript;
import static org.jclouds.crypto.CryptoStreams.base64;
import static org.jclouds.scriptbuilder.domain.Statements.exec;
import java.io.FileNotFoundException;
@ -31,7 +30,6 @@ import java.util.Map.Entry;
import org.jclouds.compute.ComputeServiceContext;
import org.jclouds.compute.ComputeServiceContextFactory;
import org.jclouds.compute.ComputeTestUtils;
import org.jclouds.compute.domain.ExecResponse;
import org.jclouds.compute.domain.NodeMetadata;
import org.jclouds.logging.log4j.config.Log4JLoggingModule;
@ -61,8 +59,7 @@ public class BYONComputeServiceLiveTest {
StringBuilder nodes = new StringBuilder();
nodes.append("nodes:\n");
nodes.append(" - id: mymachine\n");
nodes.append(" name: mymachine\n");
nodes.append(" description: my local machine\n");
nodes.append(" name: my local machine\n");
nodes.append(" hostname: localhost\n");
nodes.append(" os_arch: ").append(System.getProperty("os.arch")).append("\n");
nodes.append(" os_family: ").append(OsFamily.UNIX).append("\n");
@ -72,7 +69,7 @@ public class BYONComputeServiceLiveTest {
nodes.append(" tags:\n");
nodes.append(" - local\n");
nodes.append(" username: ").append(System.getProperty("user.name")).append("\n");
nodes.append(" credential: ").append(base64(ComputeTestUtils.setupKeyPair().get("private").getBytes()))
nodes.append(" credential_url: file://").append(System.getProperty("user.home")).append("/.ssh/id_rsa")
.append("\n");
contextProperties.setProperty("byon.nodes", nodes.toString());

12
sandbox-apis/byon/src/test/java/org/jclouds/byon/BYONComputeServiceTest.java
View File

@ -45,10 +45,18 @@ import com.google.inject.Module;
public class BYONComputeServiceTest {
@Test
public void testNodesParse() throws Exception {
public void testNodesParseWithFileUrl() throws Exception {
assertNodesParse("file://" + getClass().getResource("/test1.yaml").getPath());
}
@Test
public void testNodesParseWithClasspathUrl() throws Exception {
assertNodesParse("classpath:///test1.yaml");
}
private void assertNodesParse(String endpoint) {
ComputeServiceContext context = null;
try {
String endpoint = "file://" + getClass().getResource("/test1.yaml").getPath();
Properties props = new Properties();
props.setProperty("byon.endpoint", endpoint);

18
sandbox-apis/byon/src/test/java/org/jclouds/byon/functions/NodeToNodeMetadataTest.java
View File

@ -21,8 +21,10 @@ package org.jclouds.byon.functions;
import static org.testng.Assert.assertEquals;
import java.net.URI;
import java.util.Map;
import org.jclouds.byon.suppliers.SupplyFromProviderURIOrNodesProperty;
import org.jclouds.compute.domain.NodeMetadata;
import org.jclouds.compute.domain.NodeMetadataBuilder;
import org.jclouds.compute.domain.NodeState;
@ -46,21 +48,23 @@ import com.google.common.collect.Maps;
public class NodeToNodeMetadataTest {
public static final Location location = new LocationImpl(LocationScope.PROVIDER, "byon", "byon", null);
public static final NodeMetadata TEST1 = new NodeMetadataBuilder().ids("cluster-1").tag("hadoop").name("cluster-1").location(
location).state(NodeState.RUNNING).operatingSystem(
new OperatingSystemBuilder().name("redhat").family(OsFamily.RHEL).arch("x86").version("5.3").description(
"xyz").build()).publicAddresses(ImmutableSet.of("cluster-1.mydomain.com")).credentials(
new Credentials("myUser", "fancyfoot")).adminPassword("sudo").build();
public static final NodeMetadata TEST1 = new NodeMetadataBuilder().ids("cluster-1").tag("hadoop").name("cluster-1")
.location(location).state(NodeState.RUNNING).operatingSystem(
new OperatingSystemBuilder().name("redhat").description("redhat").family(OsFamily.RHEL)
.arch("x86").version("5.3").build()).publicAddresses(
ImmutableSet.of("cluster-1.mydomain.com")).credentials(new Credentials("myUser", "fancyfoot"))
.adminPassword("sudo").build();
@Test
public void testNodesParse() throws Exception {
Map<String, Credentials> credentialStore = Maps.newLinkedHashMap();
NodeToNodeMetadata parser = new NodeToNodeMetadata(Suppliers.ofInstance(location), credentialStore);
NodeToNodeMetadata parser = new NodeToNodeMetadata(Suppliers.ofInstance(location),
new SupplyFromProviderURIOrNodesProperty(URI.create("test")), credentialStore);
assertEquals(parser.apply(NodesFromYamlTest.TEST1), TEST1);
assertEquals(credentialStore, ImmutableMap.of("node#cluster-1", new Credentials("myUser", "fancyfoot")));
assertEquals(credentialStore, ImmutableMap.of("node#cluster-1", new Credentials("myUser", "fancyfoot")));
}
}

15
sandbox-apis/byon/src/test/java/org/jclouds/byon/functions/NodesFromYamlTest.java
View File

@ -37,9 +37,9 @@ import com.google.common.collect.ImmutableMap;
*/
public class NodesFromYamlTest {
public static final Node TEST1 = new Node("cluster-1", "cluster-1", "xyz", "cluster-1.mydomain.com", "x86", "rhel", "redhat",
"5.3", "hadoop", ImmutableList.of("vanilla"), "myUser", CryptoStreams.base64("fancyfoot".getBytes()),
CryptoStreams.base64("sudo".getBytes()));
public static final Node TEST1 = new Node("cluster-1", "cluster-1", "cluster-1.mydomain.com", "x86", "rhel",
"redhat", "5.3", "hadoop", ImmutableList.of("vanilla"), "myUser", CryptoStreams.base64("fancyfoot"
.getBytes()), null, CryptoStreams.base64("sudo".getBytes()));
@Test
public void testNodesParse() throws Exception {
@ -50,6 +50,15 @@ public class NodesFromYamlTest {
assertEquals(parser.apply(is), ImmutableMap.of(TEST1.getId(), TEST1));
}
@Test
public void testNodesParseWhenCredentialInUrl() throws Exception {
InputStream is = getClass().getResourceAsStream("/test_with_url.yaml");
NodesFromYaml parser = new NodesFromYaml();
assertEquals(parser.apply(is), ImmutableMap.of(TEST1.getId(), TEST1));
}
@Test(expectedExceptions = IllegalStateException.class)
public void testMustParseSomething() throws Exception {
new NodesFromYaml().apply(Strings2.toInputStream(""));

11
sandbox-apis/byon/src/test/java/org/jclouds/byon/suppliers/SupplyFromProviderURIOrNodesPropertyTest.java
View File

@ -44,6 +44,17 @@ public class SupplyFromProviderURIOrNodesPropertyTest {
}
@Test
public void testFromURIClasspath() throws Exception {
SupplyFromProviderURIOrNodesProperty supplier = new SupplyFromProviderURIOrNodesProperty(URI
.create("classpath:///test1.yaml"));
assertEquals(Strings2.toStringAndClose(supplier.get()), Strings2.toStringAndClose(getClass().getResourceAsStream(
"/test1.yaml")));
}
@Test
public void testFromProperty() throws Exception {

1
sandbox-apis/byon/src/test/resources/test1.yaml
View File

@ -1,7 +1,6 @@
nodes:
- id: cluster-1
name: cluster-1
description: xyz
hostname: cluster-1.mydomain.com
os_arch: x86
os_family: rhel

14
sandbox-apis/byon/src/test/resources/test_with_url.yaml Normal file
View File

@ -0,0 +1,14 @@
nodes:
- id: cluster-1
name: cluster-1
hostname: cluster-1.mydomain.com
os_arch: x86
os_family: rhel
os_name: redhat
os_version: 5.3
group: hadoop
tags:
- vanilla
username: myUser
credential_url: classpath:///testkey.txt
sudo_password: c3Vkbw==

1
sandbox-apis/byon/src/test/resources/testkey.txt Normal file
View File

@ -0,0 +1 @@
fancyfoot