From 546535340f4fa3b7df1dcb6007abeab1ed1d9d25 Mon Sep 17 00:00:00 2001 From: Adrian Cole Date: Tue, 28 Sep 2010 23:56:05 +0100 Subject: [PATCH] refactored boot script to check if java is installed first, and also stream jboss so that it works with small vms --- ...teAndBlobStoreTogetherHappilyLiveTest.java | 14 +- .../BlobStoreAndComputeServiceLiveTest.java | 21 +-- .../PlacementGroupClientLiveTest.java | 52 +++--- aws/core/src/test/resources/log4j.xml | 3 +- .../compute/BaseComputeServiceLiveTest.java | 152 +++++++----------- .../org/jclouds/compute/ComputeTestUtils.java | 94 +++++++++++ .../org/jclouds/compute/RunScriptData.java | 85 +++++----- .../StubComputeServiceIntegrationTest.java | 5 +- .../test/resources/initscript_with_java.sh | 7 +- .../test/resources/initscript_with_jboss.sh | 13 +- compute/src/test/resources/runscript.sh | 7 +- .../jclouds/scriptbuilder/InitBuilder.java | 3 +- 12 files changed, 256 insertions(+), 200 deletions(-) create mode 100644 compute/src/test/java/org/jclouds/compute/ComputeTestUtils.java diff --git a/aws/core/src/test/java/org/jclouds/aws/ComputeAndBlobStoreTogetherHappilyLiveTest.java b/aws/core/src/test/java/org/jclouds/aws/ComputeAndBlobStoreTogetherHappilyLiveTest.java index dc2da3657e..aac06fb99e 100644 --- a/aws/core/src/test/java/org/jclouds/aws/ComputeAndBlobStoreTogetherHappilyLiveTest.java +++ b/aws/core/src/test/java/org/jclouds/aws/ComputeAndBlobStoreTogetherHappilyLiveTest.java @@ -19,7 +19,7 @@ package org.jclouds.aws; -import static org.jclouds.compute.BaseComputeServiceLiveTest.buildScript; +import static org.jclouds.compute.ComputeTestUtils.buildScript; import static org.jclouds.compute.options.TemplateOptions.Builder.runScript; import static org.jclouds.compute.util.ComputeServiceUtils.execHttpResponse; import static org.jclouds.compute.util.ComputeServiceUtils.extractTargzIntoDirectory; @@ -74,7 +74,7 @@ public class ComputeAndBlobStoreTogetherHappilyLiveTest extends BlobStoreAndComp public void testWeCanIndirectBootstrapInstructionsToAnArbitraryAndPrivateBlobStore() throws RunNodesException { OperatingSystem defaultOperatingSystem = computeContext.getComputeService().templateBuilder().build().getImage() - .getOperatingSystem(); + .getOperatingSystem(); // using jclouds ability to detect operating systems before we launch them, we can avoid // the bad practice of assuming everything is ubuntu. @@ -89,22 +89,22 @@ public class ComputeAndBlobStoreTogetherHappilyLiveTest extends BlobStoreAndComp // if we want to, we can mix and match batched and ad-hoc commands, such as extracting maven String mavenVersion = "3.0-beta-3"; - Statement extractMavenIntoUsrLocal = extractTargzIntoDirectory(URI - .create("http://mirrors.ibiblio.org/pub/mirrors/apache//maven/binaries/apache-maven-" + mavenVersion - + "-bin.tar.gz"), "/usr/local"); + Statement extractMavenIntoUsrLocal = extractTargzIntoDirectory( + URI.create("http://mirrors.ibiblio.org/pub/mirrors/apache//maven/binaries/apache-maven-" + mavenVersion + + "-bin.tar.gz"), "/usr/local"); // have both of these commands occur on boot Statement bootstrapInstructions = newStatementList(installOpenJDK, extractMavenIntoUsrLocal); // now that we have the correct instructions, kick-off the provisioner Iterable nodes = computeContext.getComputeService().runNodesWithTag(tag, 2, - runScript(bootstrapInstructions)); + runScript(bootstrapInstructions)); // ensure the bootstrap operated by checking for the components we installed at boot time. // Note this test will ensure both nodes are in sync. assertSshOutputOfCommandContains(nodes, "java -version", "OpenJDK"); assertSshOutputOfCommandContains(nodes, "/usr/local/apache-maven-" + mavenVersion + "/bin/mvn -version", - "Apache Maven " + mavenVersion + ""); + "Apache Maven " + mavenVersion + ""); } } diff --git a/aws/core/src/test/java/org/jclouds/aws/ec2/compute/BlobStoreAndComputeServiceLiveTest.java b/aws/core/src/test/java/org/jclouds/aws/ec2/compute/BlobStoreAndComputeServiceLiveTest.java index ed9d68fb4b..5dbed6d16f 100644 --- a/aws/core/src/test/java/org/jclouds/aws/ec2/compute/BlobStoreAndComputeServiceLiveTest.java +++ b/aws/core/src/test/java/org/jclouds/aws/ec2/compute/BlobStoreAndComputeServiceLiveTest.java @@ -21,6 +21,7 @@ package org.jclouds.aws.ec2.compute; import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.collect.Iterables.get; +import static org.jclouds.compute.ComputeTestUtils.setupKeyPair; import java.io.FileNotFoundException; import java.io.IOException; @@ -31,7 +32,6 @@ import org.jclouds.Constants; import org.jclouds.blobstore.BlobStoreContext; import org.jclouds.blobstore.BlobStoreContextFactory; import org.jclouds.blobstore.domain.Blob; -import org.jclouds.compute.BaseComputeServiceLiveTest; import org.jclouds.compute.ComputeServiceContext; import org.jclouds.compute.ComputeServiceContextFactory; import org.jclouds.compute.domain.NodeMetadata; @@ -62,9 +62,9 @@ public class BlobStoreAndComputeServiceLiveTest { protected Properties setupCredentials(String provider) { String identity = checkNotNull(System.getProperty("test." + provider + ".identity"), "test." + provider - + ".identity"); + + ".identity"); String credential = checkNotNull(System.getProperty("test." + provider + ".credential"), "test." + provider - + ".credential"); + + ".credential"); String endpoint = System.getProperty("test." + provider + ".endpoint"); String apiversion = System.getProperty("test." + provider + ".apiversion"); Properties overrides = new Properties(); @@ -80,16 +80,17 @@ public class BlobStoreAndComputeServiceLiveTest { } protected void setupKeyPairForTest() throws FileNotFoundException, IOException { - keyPair = BaseComputeServiceLiveTest.setupKeyPair(); + keyPair = setupKeyPair(); } @BeforeGroups(groups = { "live" }) public void setupClient() throws FileNotFoundException, IOException { setupKeyPairForTest(); - computeContext = new ComputeServiceContextFactory().createContext(computeServiceProvider, ImmutableSet.of( - new Log4JLoggingModule(), new JschSshClientModule()), setupCredentials(computeServiceProvider)); - blobContext = new BlobStoreContextFactory().createContext(blobStoreProvider, ImmutableSet - .of(new Log4JLoggingModule()), setupCredentials(blobStoreProvider)); + computeContext = new ComputeServiceContextFactory().createContext(computeServiceProvider, + ImmutableSet.of(new Log4JLoggingModule(), new JschSshClientModule()), + setupCredentials(computeServiceProvider)); + blobContext = new BlobStoreContextFactory().createContext(blobStoreProvider, + ImmutableSet.of(new Log4JLoggingModule()), setupCredentials(blobStoreProvider)); blobContext.getAsyncBlobStore().createContainerInLocation(null, tag); computeContext.getComputeService().destroyNodesMatching(NodePredicates.withTag(tag)); } @@ -98,8 +99,8 @@ public class BlobStoreAndComputeServiceLiveTest { for (NodeMetadata node : nodes) { IPSocket socket = new IPSocket(get(node.getPublicAddresses(), 0), 22); - SshClient ssh = computeContext.utils().sshFactory().create(socket, node.getCredentials().identity, - node.getCredentials().credential.getBytes()); + SshClient ssh = computeContext.utils().sshFactory() + .create(socket, node.getCredentials().identity, node.getCredentials().credential.getBytes()); try { ssh.connect(); ExecResponse exec = ssh.exec(cmd); diff --git a/aws/core/src/test/java/org/jclouds/aws/ec2/services/PlacementGroupClientLiveTest.java b/aws/core/src/test/java/org/jclouds/aws/ec2/services/PlacementGroupClientLiveTest.java index fb25ba9e7a..36b3813a2b 100644 --- a/aws/core/src/test/java/org/jclouds/aws/ec2/services/PlacementGroupClientLiveTest.java +++ b/aws/core/src/test/java/org/jclouds/aws/ec2/services/PlacementGroupClientLiveTest.java @@ -20,7 +20,13 @@ package org.jclouds.aws.ec2.services; import static com.google.common.base.Preconditions.checkNotNull; -import static org.jclouds.io.Payloads.newStringPayload; +import static com.google.common.collect.Iterables.any; +import static com.google.common.collect.Iterables.getOnlyElement; +import static com.google.common.collect.Lists.newArrayList; +import static com.google.common.collect.Sets.newTreeSet; +import static org.jclouds.compute.ComputeTestUtils.buildScript; +import static org.jclouds.compute.ComputeTestUtils.setupKeyPair; +import static org.jclouds.scriptbuilder.domain.Statements.exec; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotNull; @@ -41,7 +47,6 @@ import org.jclouds.aws.ec2.domain.PlacementGroup; import org.jclouds.aws.ec2.domain.PlacementGroup.State; import org.jclouds.aws.ec2.predicates.PlacementGroupAvailable; import org.jclouds.aws.ec2.predicates.PlacementGroupDeleted; -import org.jclouds.compute.BaseComputeServiceLiveTest; import org.jclouds.compute.ComputeServiceContext; import org.jclouds.compute.ComputeServiceContextFactory; import org.jclouds.compute.RunNodesException; @@ -59,9 +64,6 @@ import org.testng.annotations.Test; import com.google.common.base.Predicate; import com.google.common.base.Throwables; import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Iterables; -import com.google.common.collect.Lists; -import com.google.common.collect.Sets; import com.google.inject.Module; /** @@ -87,10 +89,10 @@ public class PlacementGroupClientLiveTest { protected void setupCredentials() { identity = checkNotNull(System.getProperty("test." + provider + ".identity"), "test." + provider + ".identity"); credential = checkNotNull(System.getProperty("test." + provider + ".credential"), "test." + provider - + ".credential"); + + ".credential"); endpoint = checkNotNull(System.getProperty("test." + provider + ".endpoint"), "test." + provider + ".endpoint"); apiversion = checkNotNull(System.getProperty("test." + provider + ".apiversion"), "test." + provider - + ".apiversion"); + + ".apiversion"); } protected Properties setupProperties() { @@ -108,35 +110,35 @@ public class PlacementGroupClientLiveTest { public void setupClient() throws FileNotFoundException, IOException { setupCredentials(); Properties overrides = setupProperties(); - context = new ComputeServiceContextFactory().createContext(provider, ImmutableSet - . of(new Log4JLoggingModule()), overrides); - keyPair = BaseComputeServiceLiveTest.setupKeyPair(); + context = new ComputeServiceContextFactory().createContext(provider, + ImmutableSet. of(new Log4JLoggingModule()), overrides); + keyPair = setupKeyPair(); client = EC2Client.class.cast(context.getProviderSpecificContext().getApi()); availableTester = new RetryablePredicate(new PlacementGroupAvailable(client), 60, 1, - TimeUnit.SECONDS); + TimeUnit.SECONDS); deletedTester = new RetryablePredicate(new PlacementGroupDeleted(client), 60, 1, TimeUnit.SECONDS); } @Test void testDescribe() { - for (String region : Lists.newArrayList(Region.US_EAST_1)) { - SortedSet allResults = Sets.newTreeSet(client.getPlacementGroupServices() - .describePlacementGroupsInRegion(region)); + for (String region : newArrayList(Region.US_EAST_1)) { + SortedSet allResults = newTreeSet(client.getPlacementGroupServices() + .describePlacementGroupsInRegion(region)); assertNotNull(allResults); if (allResults.size() >= 1) { PlacementGroup group = allResults.last(); - SortedSet result = Sets.newTreeSet(client.getPlacementGroupServices() - .describePlacementGroupsInRegion(region, group.getName())); + SortedSet result = newTreeSet(client.getPlacementGroupServices() + .describePlacementGroupsInRegion(region, group.getName())); assertNotNull(result); PlacementGroup compare = result.last(); assertEquals(compare, group); } } - for (String region : Lists.newArrayList(Region.EU_WEST_1, Region.US_WEST_1, Region.AP_SOUTHEAST_1)) { + for (String region : newArrayList(Region.EU_WEST_1, Region.US_WEST_1, Region.AP_SOUTHEAST_1)) { try { client.getPlacementGroupServices().describePlacementGroupsInRegion(region); assert false : "should be unsupported"; @@ -158,7 +160,7 @@ public class PlacementGroupClientLiveTest { private void verifyPlacementGroup(String groupName) { assert availableTester.apply(new PlacementGroup(Region.US_EAST_1, groupName, "cluster", State.PENDING)) : group; Set oneResult = client.getPlacementGroupServices().describePlacementGroupsInRegion(null, - groupName); + groupName); assertNotNull(oneResult); assertEquals(oneResult.size(), 1); group = oneResult.iterator().next(); @@ -169,7 +171,7 @@ public class PlacementGroupClientLiveTest { public void testStartCCInstance() throws Exception { Set sizes = context.getComputeService().listHardwareProfiles(); - assert Iterables.any(sizes, new Predicate() { + assert any(sizes, new Predicate() { @Override public boolean apply(Hardware arg0) { @@ -178,7 +180,7 @@ public class PlacementGroupClientLiveTest { }) : sizes; Set images = context.getComputeService().listImages(); - assert Iterables.any(images, new Predicate() { + assert any(images, new Predicate() { @Override public boolean apply(Image arg0) { @@ -193,19 +195,17 @@ public class PlacementGroupClientLiveTest { assertEquals(template.getImage().getId(), "us-east-1/ami-7ea24a17"); template.getOptions().installPrivateKey(keyPair.get("private")).authorizePublicKey(keyPair.get("public")) - .runScript( - newStringPayload(BaseComputeServiceLiveTest.buildScript(template.getImage() - .getOperatingSystem()))); + .runScript(exec(buildScript(template.getImage().getOperatingSystem()))); String tag = PREFIX + "cccluster"; context.getComputeService().destroyNodesMatching(NodePredicates.withTag(tag)); try { Set nodes = context.getComputeService().runNodesWithTag(tag, 1, template); - NodeMetadata node = Iterables.getOnlyElement(nodes); + NodeMetadata node = getOnlyElement(nodes); - Iterables.getOnlyElement(Iterables.getOnlyElement(client.getInstanceServices().describeInstancesInRegion(null, - node.getProviderId()))); + getOnlyElement(getOnlyElement(client.getInstanceServices().describeInstancesInRegion(null, + node.getProviderId()))); } catch (RunNodesException e) { System.err.println(e.getNodeErrors().keySet()); diff --git a/aws/core/src/test/resources/log4j.xml b/aws/core/src/test/resources/log4j.xml index aee54728e8..1d36e512fd 100644 --- a/aws/core/src/test/resources/log4j.xml +++ b/aws/core/src/test/resources/log4j.xml @@ -162,11 +162,12 @@ - + diff --git a/compute/src/test/java/org/jclouds/compute/BaseComputeServiceLiveTest.java b/compute/src/test/java/org/jclouds/compute/BaseComputeServiceLiveTest.java index dc39ce9404..2beda76590 100755 --- a/compute/src/test/java/org/jclouds/compute/BaseComputeServiceLiveTest.java +++ b/compute/src/test/java/org/jclouds/compute/BaseComputeServiceLiveTest.java @@ -30,6 +30,7 @@ import static com.google.common.collect.Maps.newLinkedHashMap; import static com.google.common.collect.Maps.uniqueIndex; import static com.google.common.collect.Sets.filter; import static com.google.common.collect.Sets.newTreeSet; +import static org.jclouds.compute.ComputeTestUtils.buildScript; import static org.jclouds.compute.options.TemplateOptions.Builder.blockOnComplete; import static org.jclouds.compute.options.TemplateOptions.Builder.overrideCredentialsWith; import static org.jclouds.compute.predicates.NodePredicates.TERMINATED; @@ -41,10 +42,8 @@ import static org.jclouds.io.Payloads.newStringPayload; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotNull; -import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; -import java.net.URI; import java.util.Collection; import java.util.Map; import java.util.NoSuchElementException; @@ -67,7 +66,6 @@ import org.jclouds.compute.domain.OperatingSystem; import org.jclouds.compute.domain.Template; import org.jclouds.compute.domain.TemplateBuilder; import org.jclouds.compute.options.TemplateOptions; -import org.jclouds.compute.predicates.OperatingSystemPredicates; import org.jclouds.domain.Credentials; import org.jclouds.domain.Location; import org.jclouds.domain.LocationScope; @@ -79,16 +77,12 @@ import org.jclouds.rest.AuthorizationException; import org.jclouds.ssh.ExecResponse; import org.jclouds.ssh.SshClient; import org.jclouds.ssh.SshException; -import org.jclouds.util.Utils; import org.testng.annotations.AfterTest; import org.testng.annotations.BeforeGroups; import org.testng.annotations.Test; -import com.google.common.base.Charsets; import com.google.common.base.Function; -import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; -import com.google.common.io.Files; import com.google.inject.Guice; import com.google.inject.Module; @@ -119,21 +113,7 @@ public abstract class BaseComputeServiceLiveTest { } protected void setupKeyPairForTest() throws FileNotFoundException, IOException { - keyPair = setupKeyPair(); - } - - public static Map setupKeyPair() throws FileNotFoundException, IOException { - String secretKeyFile; - try { - secretKeyFile = checkNotNull(System.getProperty("test.ssh.keyfile"), "test.ssh.keyfile"); - } catch (NullPointerException e) { - secretKeyFile = System.getProperty("user.home") + "/.ssh/id_rsa"; - } - checkSecretKeyFile(secretKeyFile); - String secret = Files.toString(new File(secretKeyFile), Charsets.UTF_8); - assert secret.startsWith("-----BEGIN RSA PRIVATE KEY-----") : "invalid key:\n" + secret; - return ImmutableMap. of("private", secret, "public", Files.toString(new File(secretKeyFile - + ".pub"), Charsets.UTF_8)); + keyPair = ComputeTestUtils.setupKeyPair(); } protected String provider; @@ -178,18 +158,11 @@ public abstract class BaseComputeServiceLiveTest { if (context != null) context.close(); Properties props = setupProperties(); - context = new ComputeServiceContextFactory().createContext(provider, ImmutableSet.of(new Log4JLoggingModule(), - getSshModule()), props); + context = new ComputeServiceContextFactory().createContext(provider, + ImmutableSet.of(new Log4JLoggingModule(), getSshModule()), props); client = context.getComputeService(); } - private static void checkSecretKeyFile(String secretKeyFile) throws FileNotFoundException { - Utils.checkNotEmpty(secretKeyFile, "System property: [test.ssh.keyfile] set to an empty string"); - if (!new File(secretKeyFile).exists()) { - throw new FileNotFoundException("secretKeyFile not found at: " + secretKeyFile); - } - } - abstract protected Module getSshModule(); // wait up to 5 seconds for an auth exception @@ -197,8 +170,8 @@ public abstract class BaseComputeServiceLiveTest { public void testCorrectAuthException() throws Exception { ComputeServiceContext context = null; try { - context = new ComputeServiceContextFactory().createContext(provider, "MOMMA", "MIA", ImmutableSet - . of(new Log4JLoggingModule())); + context = new ComputeServiceContextFactory().createContext(provider, "MOMMA", "MIA", + ImmutableSet. of(new Log4JLoggingModule())); context.getComputeService().listNodes(); } catch (AuthorizationException e) { throw e; @@ -241,7 +214,7 @@ public abstract class BaseComputeServiceLiveTest { OperatingSystem os = get(nodes, 0).getOperatingSystem(); try { Map responses = runScriptWithCreds(tag, os, new Credentials( - good.identity, "romeo")); + good.identity, "romeo")); assert false : "shouldn't pass with a bad password\n" + responses; } catch (RunScriptOnNodesException e) { assert getRootCause(e).getMessage().contains("Auth fail") : e; @@ -273,25 +246,24 @@ public abstract class BaseComputeServiceLiveTest { } - template = client.templateBuilder().options(blockOnComplete(false).blockOnPort(8080, 300).inboundPorts(22, 8080)) - .build(); + template = client.templateBuilder().options(blockOnComplete(false).blockOnPort(8080, 600).inboundPorts(22, 8080)) + .build(); // note this is a dependency on the template resolution template.getOptions().runScript( - RunScriptData.createScriptInstallAndStartJBoss(keyPair.get("public"), template.getImage() - .getOperatingSystem())); + RunScriptData.createScriptInstallAndStartJBoss(keyPair.get("public"), template.getImage() + .getOperatingSystem())); try { - TreeSet nodes = newTreeSet(client.runNodesWithTag(tag, 1, template)); + NodeMetadata node = getOnlyElement(client.runNodesWithTag(tag, 1, template)); - checkHttpGet(nodes); + checkHttpGet(node); } finally { client.destroyNodesMatching(withTag(tag)); } } - protected void checkHttpGet(TreeSet nodes) { - assert context.utils().http().get( - URI.create(String.format("http://%s:8080", get(nodes.last().getPublicAddresses(), 0)))) != null; + protected void checkHttpGet(NodeMetadata node) { + ComputeTestUtils.checkHttpGet(context.utils().http(), node, 8080); } @Test(enabled = true, dependsOnMethods = "testCreateAndRunAService") @@ -327,7 +299,7 @@ public abstract class BaseComputeServiceLiveTest { template = buildTemplate(client.templateBuilder()); template.getOptions().installPrivateKey(keyPair.get("private")).authorizePublicKey(keyPair.get("public")) - .runScript(newStringPayload(buildScript(template.getImage().getOperatingSystem()))); + .runScript(newStringPayload(buildScript(template.getImage().getOperatingSystem()))); } protected void checkImageIdMatchesTemplate(NodeMetadata node) { @@ -338,8 +310,8 @@ public abstract class BaseComputeServiceLiveTest { protected void checkOsMatchesTemplate(NodeMetadata node) { if (node.getOperatingSystem() != null) assert node.getOperatingSystem().getFamily().equals(template.getImage().getOperatingSystem().getFamily()) : String - .format("expecting family %s but got %s", template.getImage().getOperatingSystem().getFamily(), node - .getOperatingSystem()); + .format("expecting family %s but got %s", template.getImage().getOperatingSystem().getFamily(), + node.getOperatingSystem()); } void assertLocationSameOrChild(Location test, Location expected) { @@ -364,10 +336,10 @@ public abstract class BaseComputeServiceLiveTest { } protected Map runScriptWithCreds(final String tag, OperatingSystem os, - Credentials creds) throws RunScriptOnNodesException { + Credentials creds) throws RunScriptOnNodesException { try { return client.runScriptOnNodesMatching(runningWithTag(tag), newStringPayload(buildScript(os)), - overrideCredentialsWith(creds).nameTask("runScriptWithCreds")); + overrideCredentialsWith(creds).nameTask("runScriptWithCreds")); } catch (SshException e) { throw e; } @@ -393,29 +365,18 @@ public abstract class BaseComputeServiceLiveTest { return templateBuilder.build(); } - public static String buildScript(OperatingSystem os) { - if (OperatingSystemPredicates.supportsApt().apply(os)) - return RunScriptData.APT_RUN_SCRIPT; - else if (OperatingSystemPredicates.supportsYum().apply(os)) - return RunScriptData.YUM_RUN_SCRIPT; - else if (OperatingSystemPredicates.supportsZypper().apply(os)) - return RunScriptData.ZYPPER_RUN_SCRIPT; - else - throw new IllegalArgumentException("don't know how to handle" + os.toString()); - } - @Test(enabled = true, dependsOnMethods = "testCreateAnotherNodeWithANewContextToEnsureSharedMemIsntRequired") public void testGet() throws Exception { - Map metadataMap = newLinkedHashMap(uniqueIndex(filter(client - .listNodesDetailsMatching(all()), and(withTag(tag), not(TERMINATED))), - new Function() { + Map metadataMap = newLinkedHashMap(uniqueIndex( + filter(client.listNodesDetailsMatching(all()), and(withTag(tag), not(TERMINATED))), + new Function() { - @Override - public String apply(NodeMetadata from) { - return from.getId(); - } + @Override + public String apply(NodeMetadata from) { + return from.getId(); + } - })); + })); for (NodeMetadata node : nodes) { metadataMap.remove(node.getId()); NodeMetadata metadata = client.getNodeMetadata(node.getId()); @@ -433,7 +394,7 @@ public abstract class BaseComputeServiceLiveTest { protected void assertNodeZero(Collection metadataSet) { assert metadataSet.size() == 0 : String.format("nodes left in set: [%s] which didn't match set: [%s]", - metadataSet, nodes); + metadataSet, nodes); } @Test(enabled = true, dependsOnMethods = "testGet") @@ -494,26 +455,26 @@ public abstract class BaseComputeServiceLiveTest { assert location != location.getParent() : location; assert location.getScope() != null : location; switch (location.getScope()) { - case PROVIDER: - assertProvider(location); - break; - case REGION: - assertProvider(location.getParent()); - break; - case ZONE: - Location provider = location.getParent().getParent(); - // zone can be a direct descendant of provider - if (provider == null) - provider = location.getParent(); - assertProvider(provider); - break; - case HOST: - Location provider2 = location.getParent().getParent().getParent(); - // zone can be a direct descendant of provider - if (provider2 == null) - provider2 = location.getParent().getParent(); - assertProvider(provider2); - break; + case PROVIDER: + assertProvider(location); + break; + case REGION: + assertProvider(location.getParent()); + break; + case ZONE: + Location provider = location.getParent().getParent(); + // zone can be a direct descendant of provider + if (provider == null) + provider = location.getParent(); + assertProvider(provider); + break; + case HOST: + Location provider2 = location.getParent().getParent().getParent(); + // zone can be a direct descendant of provider + if (provider2 == null) + provider2 = location.getParent().getParent(); + assertProvider(provider2); + break; } } } @@ -600,18 +561,21 @@ public abstract class BaseComputeServiceLiveTest { // a socket conection // state. SshClient ssh = (node.getCredentials().credential != null && !node.getCredentials().credential - .startsWith("-----BEGIN RSA PRIVATE KEY-----")) ? context.utils().sshFactory().create(socket, - node.getCredentials().identity, node.getCredentials().credential) : context.utils().sshFactory().create( - socket, - node.getCredentials().identity, - node.getCredentials().credential != null ? node.getCredentials().credential.getBytes() : keyPair.get( + .startsWith("-----BEGIN RSA PRIVATE KEY-----")) ? context.utils().sshFactory() + .create(socket, node.getCredentials().identity, node.getCredentials().credential) : context + .utils() + .sshFactory() + .create( + socket, + node.getCredentials().identity, + node.getCredentials().credential != null ? node.getCredentials().credential.getBytes() : keyPair.get( "private").getBytes()); try { ssh.connect(); ExecResponse hello = ssh.exec("echo hello"); assertEquals(hello.getOutput().trim(), "hello"); ExecResponse exec = ssh.exec("java -version"); - assert exec.getError().indexOf("OpenJDK") != -1 || exec.getOutput().indexOf("OpenJDK") != -1 : exec; + assert exec.getError().indexOf("1.6") != -1 || exec.getOutput().indexOf("1.6") != -1 : exec; } finally { if (ssh != null) ssh.disconnect(); diff --git a/compute/src/test/java/org/jclouds/compute/ComputeTestUtils.java b/compute/src/test/java/org/jclouds/compute/ComputeTestUtils.java new file mode 100644 index 0000000000..d55634076b --- /dev/null +++ b/compute/src/test/java/org/jclouds/compute/ComputeTestUtils.java @@ -0,0 +1,94 @@ +/** + * + * Copyright (C) 2010 Cloud Conscious, LLC. + * + * ==================================================================== + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ==================================================================== + */ + +package org.jclouds.compute; + +import static org.jclouds.util.Utils.checkNotEmpty; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.lang.reflect.UndeclaredThrowableException; +import java.net.URI; +import java.util.Map; +import java.util.concurrent.TimeoutException; +import static org.testng.Assert.assertEquals; +import static com.google.common.base.Preconditions.checkNotNull; + +import org.jclouds.compute.domain.NodeMetadata; +import org.jclouds.compute.domain.OperatingSystem; +import org.jclouds.compute.predicates.OperatingSystemPredicates; +import org.jclouds.rest.HttpClient; + +import com.google.common.base.Charsets; +import com.google.common.collect.ImmutableMap; +import com.google.common.io.Files; +import static com.google.common.collect.Iterables.get; + +/** + * utilities helpful in testing compute providers + * + * @author Adrian Cole + */ +public class ComputeTestUtils { + public static String buildScript(OperatingSystem os) { + if (OperatingSystemPredicates.supportsApt().apply(os)) + return RunScriptData.APT_RUN_SCRIPT; + else if (OperatingSystemPredicates.supportsYum().apply(os)) + return RunScriptData.YUM_RUN_SCRIPT; + else if (OperatingSystemPredicates.supportsZypper().apply(os)) + return RunScriptData.ZYPPER_RUN_SCRIPT; + else + throw new IllegalArgumentException("don't know how to handle" + os.toString()); + } + + public static Map setupKeyPair() throws FileNotFoundException, IOException { + String secretKeyFile; + try { + secretKeyFile = checkNotNull(System.getProperty("test.ssh.keyfile"), "test.ssh.keyfile"); + } catch (NullPointerException e) { + secretKeyFile = System.getProperty("user.home") + "/.ssh/id_rsa"; + } + checkSecretKeyFile(secretKeyFile); + String secret = Files.toString(new File(secretKeyFile), Charsets.UTF_8); + assert secret.startsWith("-----BEGIN RSA PRIVATE KEY-----") : "invalid key:\n" + secret; + return ImmutableMap. of("private", secret, "public", + Files.toString(new File(secretKeyFile + ".pub"), Charsets.UTF_8)); + } + + public static void checkSecretKeyFile(String secretKeyFile) throws FileNotFoundException { + checkNotEmpty(secretKeyFile, "System property: [test.ssh.keyfile] set to an empty string"); + if (!new File(secretKeyFile).exists()) { + throw new FileNotFoundException("secretKeyFile not found at: " + secretKeyFile); + } + } + + public static void checkHttpGet(HttpClient client, NodeMetadata node, int port) { + for (int i = 0; i < 5; i++) + try { + assert client.get(URI.create(String.format("http://%s:%d", get(node.getPublicAddresses(), 0), port))) != null; + break; + } catch (UndeclaredThrowableException e) { + assertEquals(e.getCause().getClass(), TimeoutException.class); + try { + Thread.sleep(1000); + } catch (InterruptedException e1) { + } + } + } +} diff --git a/compute/src/test/java/org/jclouds/compute/RunScriptData.java b/compute/src/test/java/org/jclouds/compute/RunScriptData.java index 4b5c664ccb..15866678f5 100644 --- a/compute/src/test/java/org/jclouds/compute/RunScriptData.java +++ b/compute/src/test/java/org/jclouds/compute/RunScriptData.java @@ -19,7 +19,7 @@ package org.jclouds.compute; -import static org.jclouds.compute.util.ComputeServiceUtils.extractZipIntoDirectory; +import static org.jclouds.compute.util.ComputeServiceUtils.extractTargzIntoDirectory; import static org.jclouds.scriptbuilder.domain.Statements.exec; import static org.jclouds.scriptbuilder.domain.Statements.interpret; @@ -40,8 +40,7 @@ import com.google.common.collect.ImmutableMap; * @author Adrian Cole */ public class RunScriptData { - private static String jbossVersion = "5.0.0.CR2"; - private static String jboss = String.format("jboss-%s-jdk6", jbossVersion); + private static String jbossHome = "/usr/local/jboss"; public static String createScriptInstallBase(OperatingSystem os) { @@ -58,52 +57,52 @@ public class RunScriptData { public static Statement createScriptInstallAndStartJBoss(String publicKey, OperatingSystem os) { Map envVariables = ImmutableMap.of("jbossHome", jbossHome); Statement toReturn = new InitBuilder( - "jboss", - jbossHome, - jbossHome, - envVariables, - ImmutableList. of(new AuthorizeRSAPublicKey(publicKey), exec(createScriptInstallBase(os)), - extractZipIntoDirectory(URI.create(String.format( - "http://superb-sea2.dl.sourceforge.net/project/jboss/JBoss/JBoss-%s/%s.zip", - jbossVersion, jboss)), "/usr/local"), exec("{md} " + jbossHome), - exec("mv /usr/local/jboss-" + jbossVersion + "/* " + jbossHome)), - ImmutableList - . of(interpret("java -Xms128m -Xmx512m -XX:MaxPermSize=256m -Dorg.jboss.resolver.warning=true -Dsun.rmi.dgc.client.gcInterval=3600000 -Dsun.rmi.dgc.server.gcInterval=3600000 -Djava.endorsed.dirs=lib/endorsed -classpath bin/run.jar org.jboss.Main -b 0.0.0.0"))); + "jboss", + jbossHome, + jbossHome, + envVariables, + ImmutableList. of( + new AuthorizeRSAPublicKey(publicKey), + exec(createScriptInstallBase(os)), + exec("rm -rf /var/cache/apt /usr/lib/vmware-tools"),// jeos hasn't enough room! + extractTargzIntoDirectory( + URI.create("http://commondatastorage.googleapis.com/jclouds-repo/jboss-as-distribution-6.0.0.20100911-M5.tar.gz"), + "/usr/local"), exec("{md} " + jbossHome), exec("mv /usr/local/jboss-*/* " + jbossHome), + exec("chmod -R oug+r+w " + jbossHome)), + ImmutableList + . of(interpret("java -Xms128m -Xmx512m -XX:MaxPermSize=256m -Dorg.jboss.resolver.warning=true -Dsun.rmi.dgc.client.gcInterval=3600000 -Dsun.rmi.dgc.server.gcInterval=3600000 -Djava.endorsed.dirs=lib/endorsed -classpath bin/run.jar org.jboss.Main -b 0.0.0.0"))); return toReturn; } public static final String APT_RUN_SCRIPT = new StringBuilder()// - .append("echo nameserver 208.67.222.222 >> /etc/resolv.conf\n")// - .append("cp /etc/apt/sources.list /etc/apt/sources.list.old\n")// - .append( - "sed 's~us.archive.ubuntu.com~mirror.anl.gov/pub~g' /etc/apt/sources.list.old >/etc/apt/sources.list\n")// - .append("apt-get update -y -qq\n")// - .append("apt-get install -f -y -qq --force-yes curl\n")// - .append("apt-get install -f -y -qq --force-yes unzip\n")// - .append("apt-get install -f -y -qq --force-yes openjdk-6-jdk\n")// - .toString(); + .append("echo nameserver 208.67.222.222 >> /etc/resolv.conf\n")// + .append("cp /etc/apt/sources.list /etc/apt/sources.list.old\n")// + .append( + "sed 's~us.archive.ubuntu.com~mirror.anl.gov/pub~g' /etc/apt/sources.list.old >/etc/apt/sources.list\n")// + .append("which curl || apt-get update -y -qq && apt-get install -f -y -qq --force-yes curl\n")// + .append( + "(which java && java -fullversion 2>&1|egrep -q 1.6 ) || apt-get install -f -y -qq --force-yes openjdk-6-jre\n")// + .append("rm -rf /var/cache/apt /usr/lib/vmware-tools\n")// jeos hasn't enough room! + .toString(); public static final String YUM_RUN_SCRIPT = new StringBuilder() - .append("echo nameserver 208.67.222.222 >> /etc/resolv.conf\n") - // - .append("echo \"[jdkrepo]\" >> /etc/yum.repos.d/CentOS-Base.repo\n") - // - .append("echo \"name=jdkrepository\" >> /etc/yum.repos.d/CentOS-Base.repo\n") - // - .append( - "echo \"baseurl=http://ec2-us-east-mirror.rightscale.com/epel/5/i386/\" >> /etc/yum.repos.d/CentOS-Base.repo\n")// - .append("echo \"enabled=1\" >> /etc/yum.repos.d/CentOS-Base.repo\n")// - .append("yum --nogpgcheck -y install unzip\n")// - .append("yum --nogpgcheck -y install curl\n")// - .append("yum --nogpgcheck -y install java-1.6.0-openjdk\n")// - .append("echo \"export PATH=\\\"/usr/lib/jvm/jre-1.6.0-openjdk/bin/:\\$PATH\\\"\" >> /root/.bashrc\n")// - .toString(); + .append("echo nameserver 208.67.222.222 >> /etc/resolv.conf\n") // + .append("echo \"[jdkrepo]\" >> /etc/yum.repos.d/CentOS-Base.repo\n") // + .append("echo \"name=jdkrepository\" >> /etc/yum.repos.d/CentOS-Base.repo\n") // + .append( + "echo \"baseurl=http://ec2-us-east-mirror.rightscale.com/epel/5/i386/\" >> /etc/yum.repos.d/CentOS-Base.repo\n")// + .append("echo \"enabled=1\" >> /etc/yum.repos.d/CentOS-Base.repo\n")// + // .append("which unzip ||yum --nogpgcheck -y install unzip\n")// + .append("which curl ||yum --nogpgcheck -y install curl\n")// + .append( + "(which java && java -fullversion 2>&1|egrep -q 1.6 ) || yum --nogpgcheck -y install java-1.6.0-openjdk&&")// + .append("echo \"export PATH=\\\"/usr/lib/jvm/jre-1.6.0-openjdk/bin/:\\$PATH\\\"\" >> /root/.bashrc\n")// + .toString(); public static final String ZYPPER_RUN_SCRIPT = new StringBuilder()// - .append("echo nameserver 208.67.222.222 >> /etc/resolv.conf\n")// - .append("sudo zypper install unzip\n")// - .append("sudo zypper install curl\n")// - .append("sudo zypper install java-1.6.0-openjdk-devl\n")// - .toString(); - + .append("echo nameserver 208.67.222.222 >> /etc/resolv.conf\n")// + // .append("which unzip ||sudo zypper install unzip\n")// + .append("which curl || zypper install curl\n")// + .append("(which java && java -fullversion 2>&1|egrep -q 1.6 ) || zypper install java-1.6.0-openjdk\n")// + .toString(); } diff --git a/compute/src/test/java/org/jclouds/compute/StubComputeServiceIntegrationTest.java b/compute/src/test/java/org/jclouds/compute/StubComputeServiceIntegrationTest.java index 8830b4b14c..16f7c1446f 100644 --- a/compute/src/test/java/org/jclouds/compute/StubComputeServiceIntegrationTest.java +++ b/compute/src/test/java/org/jclouds/compute/StubComputeServiceIntegrationTest.java @@ -32,7 +32,6 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.io.Serializable; import java.util.Set; -import java.util.TreeSet; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; @@ -105,7 +104,7 @@ public class StubComputeServiceIntegrationTest extends BaseComputeServiceLiveTes } @Override - protected void checkHttpGet(TreeSet nodes) { + protected void checkHttpGet(NodeMetadata node) { } @@ -228,7 +227,7 @@ public class StubComputeServiceIntegrationTest extends BaseComputeServiceLiveTes client.connect(); expect(client.exec("echo hello")).andReturn(new ExecResponse("hello", "", 0)); - expect(client.exec("java -version")).andReturn(new ExecResponse("", "OpenJDK", 0)); + expect(client.exec("java -version")).andReturn(new ExecResponse("", "1.6", 0)); client.disconnect(); } diff --git a/compute/src/test/resources/initscript_with_java.sh b/compute/src/test/resources/initscript_with_java.sh index df87d2435d..fb354f897f 100644 --- a/compute/src/test/resources/initscript_with_java.sh +++ b/compute/src/test/resources/initscript_with_java.sh @@ -79,10 +79,9 @@ cd $INSTANCE_HOME echo nameserver 208.67.222.222 >> /etc/resolv.conf cp /etc/apt/sources.list /etc/apt/sources.list.old sed 's~us.archive.ubuntu.com~mirror.anl.gov/pub~g' /etc/apt/sources.list.old >/etc/apt/sources.list -apt-get update -y -qq -apt-get install -f -y -qq --force-yes curl -apt-get install -f -y -qq --force-yes unzip -apt-get install -f -y -qq --force-yes openjdk-6-jdk +which curl || apt-get update -y -qq && apt-get install -f -y -qq --force-yes curl +(which java && java -fullversion 2>&1|egrep -q 1.6 ) || apt-get install -f -y -qq --force-yes openjdk-6-jre +rm -rf /var/cache/apt /usr/lib/vmware-tools mkdir -p ~/.ssh cat >> ~/.ssh/authorized_keys <<'END_OF_FILE' diff --git a/compute/src/test/resources/initscript_with_jboss.sh b/compute/src/test/resources/initscript_with_jboss.sh index dc94b9b7ac..1640f77670 100644 --- a/compute/src/test/resources/initscript_with_jboss.sh +++ b/compute/src/test/resources/initscript_with_jboss.sh @@ -66,14 +66,15 @@ END_OF_FILE echo nameserver 208.67.222.222 >> /etc/resolv.conf cp /etc/apt/sources.list /etc/apt/sources.list.old sed 's~us.archive.ubuntu.com~mirror.anl.gov/pub~g' /etc/apt/sources.list.old >/etc/apt/sources.list - apt-get update -y -qq - apt-get install -f -y -qq --force-yes curl - apt-get install -f -y -qq --force-yes unzip - apt-get install -f -y -qq --force-yes openjdk-6-jdk + which curl || apt-get update -y -qq && apt-get install -f -y -qq --force-yes curl + (which java && java -fullversion 2>&1|egrep -q 1.6 ) || apt-get install -f -y -qq --force-yes openjdk-6-jre + rm -rf /var/cache/apt /usr/lib/vmware-tools - (mkdir -p /usr/local &&cd /usr/local &&curl -X GET -s --retry 20 http://superb-sea2.dl.sourceforge.net/project/jboss/JBoss/JBoss-5.0.0.CR2/jboss-5.0.0.CR2-jdk6.zip >extract.zip && unzip -o -qq extract.zip&& rm extract.zip) + rm -rf /var/cache/apt /usr/lib/vmware-tools + curl -X GET -s --retry 20 http://commondatastorage.googleapis.com/jclouds-repo/jboss-as-distribution-6.0.0.20100911-M5.tar.gz |(mkdir -p /usr/local &&cd /usr/local &&tar -xpzf -) mkdir -p /usr/local/jboss - mv /usr/local/jboss-5.0.0.CR2/* /usr/local/jboss + mv /usr/local/jboss-*/* /usr/local/jboss + chmod -R oug+r+w /usr/local/jboss mkdir -p $INSTANCE_HOME # create runscript header diff --git a/compute/src/test/resources/runscript.sh b/compute/src/test/resources/runscript.sh index 84f034da27..760f231c90 100644 --- a/compute/src/test/resources/runscript.sh +++ b/compute/src/test/resources/runscript.sh @@ -79,10 +79,9 @@ cd $INSTANCE_HOME echo nameserver 208.67.222.222 >> /etc/resolv.conf cp /etc/apt/sources.list /etc/apt/sources.list.old sed 's~us.archive.ubuntu.com~mirror.anl.gov/pub~g' /etc/apt/sources.list.old >/etc/apt/sources.list -apt-get update -y -qq -apt-get install -f -y -qq --force-yes curl -apt-get install -f -y -qq --force-yes unzip -apt-get install -f -y -qq --force-yes openjdk-6-jdk +which curl || apt-get update -y -qq && apt-get install -f -y -qq --force-yes curl +(which java && java -fullversion 2>&1|egrep -q 1.6 ) || apt-get install -f -y -qq --force-yes openjdk-6-jre +rm -rf /var/cache/apt /usr/lib/vmware-tools END_OF_SCRIPT diff --git a/scriptbuilder/src/main/java/org/jclouds/scriptbuilder/InitBuilder.java b/scriptbuilder/src/main/java/org/jclouds/scriptbuilder/InitBuilder.java index 361c2aa94a..dbeb565636 100644 --- a/scriptbuilder/src/main/java/org/jclouds/scriptbuilder/InitBuilder.java +++ b/scriptbuilder/src/main/java/org/jclouds/scriptbuilder/InitBuilder.java @@ -54,7 +54,6 @@ public class InitBuilder extends ScriptBuilder { this(instanceName, instanceHome, logDir, variables, ImmutableSet.of(), statements); } - @SuppressWarnings("unchecked") public InitBuilder(String instanceName, String instanceHome, String logDir, Map variables, Iterable initStatements, Iterable statements) { this.instanceName = checkNotNull(instanceName, "instanceName"); @@ -67,7 +66,7 @@ public class InitBuilder extends ScriptBuilder { .addStatement( switchArg( 1, - new ImmutableMap.Builder() + new ImmutableMap.Builder() .put( "init", newStatementList(call("default"), call(instanceName),