first working integration between chef and the compute apis

chef/compute/pom.xml

@@ -77,6 +77,12 @@
             <artifactId>jclouds-compute</artifactId>
             <version>${project.version}</version>
         </dependency>
+        <dependency>
+            <groupId>${project.groupId}</groupId>
+            <artifactId>jclouds-jsch</artifactId>
+            <version>${project.version}</version>
+            <scope>test</scope>
+        </dependency>
         <dependency>
             <groupId>${project.groupId}</groupId>
             <artifactId>jclouds-log4j</artifactId>

chef/compute/src/main/resources/herefile.sh

@@ -1,5 +0,0 @@
-(
-cat <<'EOP'
-@herefile@
-EOP
-) > @destination@

chef/compute/src/test/java/org/jclouds/chef/compute/ChefComputeServiceLiveTest.java

@@ -21,27 +21,52 @@ package org.jclouds.chef.compute;
 
 import static com.google.common.base.Preconditions.checkNotNull;
 import static com.google.common.collect.Sets.newHashSet;
+import static org.jclouds.io.Payloads.newStringPayload;
+import static org.jclouds.scriptbuilder.domain.Statements.createFile;
+import static org.jclouds.scriptbuilder.domain.Statements.exec;
+import static org.jclouds.scriptbuilder.domain.Statements.newStatementList;
 
 import java.io.File;
+import java.io.FileNotFoundException;
 import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
 import java.util.Properties;
 import java.util.Set;
 
 import org.jclouds.chef.ChefContext;
 import org.jclouds.chef.ChefContextFactory;
-import org.jclouds.chef.ChefService;
+import org.jclouds.compute.BaseComputeServiceLiveTest;
 import org.jclouds.compute.ComputeServiceContext;
 import org.jclouds.compute.ComputeServiceContextFactory;
+import org.jclouds.compute.RunNodesException;
+import org.jclouds.compute.domain.NodeMetadata;
+import org.jclouds.compute.options.TemplateOptions;
+import org.jclouds.compute.predicates.NodePredicates;
 import org.jclouds.crypto.Pems;
+import org.jclouds.json.Json;
 import org.jclouds.logging.log4j.config.Log4JLoggingModule;
+import org.jclouds.scriptbuilder.domain.OsFamily;
+import org.jclouds.scriptbuilder.domain.Statement;
+import org.jclouds.ssh.jsch.config.JschSshClientModule;
+import org.jclouds.util.Utils;
 import org.testng.annotations.AfterGroups;
 import org.testng.annotations.BeforeGroups;
 import org.testng.annotations.Test;
 
 import com.google.common.base.Charsets;
+import com.google.common.base.Splitter;
+import com.google.common.base.Throwables;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
 import com.google.common.io.Files;
 import com.google.inject.Module;
+import com.google.inject.TypeLiteral;
 
 /**
  * 
@@ -54,9 +79,12 @@ public class ChefComputeServiceLiveTest {
    private ChefContext chefContext;
    private String tag;
    private String clientName;
+   private String chefEndpoint;
+   private Map<String, String> keyPair;
+   private Iterable<? extends NodeMetadata> nodes;
 
    @BeforeGroups(groups = { "live" })
-   public void setupCompute() {
+   public void setupCompute() throws FileNotFoundException, IOException {
       tag = System.getProperty("jclouds.compute.tag") != null ? System.getProperty("jclouds.compute.tag")
                : "jcloudschef";
       String computeProvider = checkNotNull(System.getProperty("jclouds.compute.provider"), "jclouds.compute.provider");
@@ -68,12 +96,13 @@ public class ChefComputeServiceLiveTest {
       String computeCredential = checkNotNull(System.getProperty("jclouds.compute.credential"),
                "jclouds.compute.credential");
       computeContext = new ComputeServiceContextFactory().createContext(computeProvider, computeIdentity,
-               computeCredential, ImmutableSet.of(new Log4JLoggingModule()), props);
+               computeCredential, ImmutableSet.of(new Log4JLoggingModule(), getSshModule()), props);
+      keyPair = BaseComputeServiceLiveTest.setupKeyPair();
    }
 
    @BeforeGroups(groups = { "live" })
    public void setupChef() throws IOException {
-      String chefEndpoint = checkNotNull(System.getProperty("jclouds.chef.endpoint"), "jclouds.chef.endpoint");
+      chefEndpoint = checkNotNull(System.getProperty("jclouds.chef.endpoint"), "jclouds.chef.endpoint");
       String chefIdentity = checkNotNull(System.getProperty("jclouds.chef.identity"), "jclouds.chef.identity");
       String chefCredentialFile = System.getProperty("jclouds.chef.credential.pem");
       if (chefCredentialFile == null || chefCredentialFile.equals(""))
@@ -84,27 +113,83 @@ public class ChefComputeServiceLiveTest {
                Charsets.UTF_8), ImmutableSet.<Module> of(new Log4JLoggingModule()), props);
    }
 
+   public static class InstallChefGems implements Statement {
+
+      public String render(OsFamily family) {
+         try {
+            return Utils.toStringAndClose(InstallChefGems.class.getClassLoader().getResourceAsStream(
+                     "install-chef-gems.sh"));
+         } catch (IOException e) {
+            Throwables.propagate(e);
+            return null;
+         }
+      }
+
+      @Override
+      public Iterable<String> functionDependecies(OsFamily family) {
+         return Collections.emptyList();
+      }
+
+   }
+
+   protected Module getSshModule() {
+      return new JschSshClientModule();
+   }
+
    @Test
-   public void test() throws IOException {
-      clientName = findNextClientName(chefContext, tag + "-%d");
-      String clientKey = Pems.pem(chefContext.getApi().createClient(clientName).getPrivateKey());
-
-      // herefile /etc/chef/client.rb
-      // log_level :info
-      // log_location STDOUT
-      // chef_server_url "@chef_server_url@"
-
-      // herefile /etc/chef/client.pem
-      // clientKey
-      // herefile /etc/chef/first-boot.json
-      // { "run_list": [ "recipe[apache]" ] }
-
-      // then run /usr/bin/chef-client -j /etc/chef/first-boot.json
+   public void test() throws IOException, InterruptedException {
+      clientName = findNextClientName(chefContext, tag + "-validator-%d");
 
       System.out.println("created new client: " + clientName);
 
-      computeContext.getComputeService().listNodes();
-      chefContext.getChefService().listNodesDetails();
+      List<String> runList = ImmutableList.of("recipe[apache2]");
+      Json json = computeContext.utils().json();
+
+      String clientKey = Pems.pem(chefContext.getApi().createClient(clientName).getPrivateKey());
+
+      Statement installChefGems = new InstallChefGems();
+      String chefConfigDir = "{root}etc{fs}chef";
+      Statement createChefConfigDir = exec("{md} " + chefConfigDir);
+      Statement createClientRb = createFile(chefConfigDir + "{fs}client.rb", ImmutableList.of("require 'rubygems'",
+               "require 'ohai'", "o = Ohai::System.new", "o.all_plugins", String.format(
+                        "node_name \"%s-\" + o[:ipaddress]", tag), "log_level :info", "log_location STDOUT", String
+                        .format("validation_client_name \"%s\"", clientName), String.format("chef_server_url \"%s\"",
+                        chefEndpoint)));
+
+      Statement createValidationPem = createFile(chefConfigDir + "{fs}validation.pem", Splitter.on('\n').split(
+               clientKey));
+      String chefBootFile = chefConfigDir + "{fs}first-boot.json";
+
+      Statement createFirstBoot = createFile(chefBootFile, Collections.singleton(json.toJson(ImmutableMap
+               .<String, List<String>> of("run_list", runList), new TypeLiteral<Map<String, List<String>>>() {
+      }.getType())));
+
+      Statement runChef = exec("chef-client -j " + chefBootFile);
+
+      Statement bootstrapAndRunChef = newStatementList(installChefGems, createChefConfigDir, createClientRb,
+               createValidationPem, createFirstBoot, runChef);
+
+      String runScript = bootstrapAndRunChef.render(OsFamily.UNIX);
+      System.out.println(runScript);
+
+      TemplateOptions options = computeContext.getComputeService().templateOptions().//
+               installPrivateKey(newStringPayload(keyPair.get("private"))).//
+               authorizePublicKey(newStringPayload(keyPair.get("public"))).//
+               runScript(newStringPayload(runScript));
+
+      try {
+         nodes = computeContext.getComputeService().runNodesWithTag(tag, 1, options);
+      } catch (RunNodesException e) {
+         nodes = Iterables.concat(e.getSuccessfulNodes(), e.getNodeErrors().keySet());
+      }
+
+      for (NodeMetadata node : nodes) {
+         URI uri = URI.create("http://" + Iterables.getLast(node.getPublicAddresses()));
+         InputStream content = computeContext.utils().http().get(uri);
+         String string = Utils.toStringAndClose(content);
+         assert string.indexOf("It works!") >= 0 : string;
+      }
+
    }
 
    private String findNextClientName(ChefContext context, String pattern) {
@@ -122,13 +207,16 @@ public class ChefComputeServiceLiveTest {
 
    @AfterGroups(groups = { "live" })
    public void teardownCompute() {
-      if (computeContext != null)
+      if (computeContext != null) {
+         computeContext.getComputeService().destroyNodesMatching(NodePredicates.withTag(tag));
          computeContext.close();
+      }
    }
 
    @AfterGroups(groups = { "live" })
    public void teardownChef() {
       if (chefContext != null) {
+         chefContext.getChefService().cleanupStaleNodesAndClients(tag + "-", 1);
          if (clientName != null && chefContext.getApi().clientExists(clientName))
             chefContext.getApi().deleteClient(clientName);
          chefContext.close();

chef/compute/src/test/resources/install-chef-gems.sh

@@ -4,11 +4,12 @@ if [ ! -f /usr/bin/chef-client ]; then
   mkdir -p /tmp/bootchef
   (
     cd /tmp/bootchef
-    curl http://rubyforge.org/frs/download.php/69365/rubygems-1.3.6.tgz| tar -xzf -
+    wget http://rubyforge.org/frs/download.php/69365/rubygems-1.3.6.tgz
+    tar xvf rubygems-1.3.6.tgz
     cd rubygems-1.3.6
     ruby setup.rb
     cp /usr/bin/gem1.8 /usr/bin/gem
   )
   rm -rf /tmp/bootchef
   gem install chef ohai --no-rdoc --no-ri --verbose
-fi
+fi

chef/compute/src/test/resources/log4j.xml

@@ -25,7 +25,27 @@
     -->
 <log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/"
     debug="false">
+    <!-- A time/date based rolling appender -->
+    <appender name="COMPUTEFILE" class="org.apache.log4j.DailyRollingFileAppender">
+        <param name="File" value="target/test-data/jclouds-compute.log" />
+        <param name="Append" value="true" />
 
+        <!-- Rollover at midnight each day -->
+        <param name="DatePattern" value="'.'yyyy-MM-dd" />
+
+        <param name="Threshold" value="TRACE" />
+
+        <layout class="org.apache.log4j.PatternLayout">
+            <!-- The default pattern: Date Priority [Category] Message\n -->
+            <param name="ConversionPattern" value="%d %-5p [%c] (%t) %m%n" />
+
+            <!--
+                The full pattern: Date MS Priority [Category]
+                (Thread:NDC) Message\n <param name="ConversionPattern"
+                value="%d %-5r %-5p [%c] (%t:%x) %m%n"/>
+            -->
+        </layout>
+    </appender>
     <!-- A time/date based rolling appender -->
     <appender name="SSHFILE" class="org.apache.log4j.DailyRollingFileAppender">
         <param name="File" value="target/test-data/jclouds-ssh.log" />
@@ -113,6 +133,9 @@
             -->
         </layout>
     </appender>
+    <appender name="ASYNCCOMPUTE" class="org.apache.log4j.AsyncAppender">
+        <appender-ref ref="COMPUTEFILE" />
+    </appender>
     <appender name="ASYNCCHEF" class="org.apache.log4j.AsyncAppender">
         <appender-ref ref="CHEFFILE" />
     </appender>
@@ -156,6 +179,11 @@
         <priority value="DEBUG" />
         <appender-ref ref="ASYNCSSH" />
     </category>
+
+    <category name="jclouds.compute">
+        <priority value="TRACE" />
+        <appender-ref ref="ASYNCCOMPUTE" />
+    </category>
     <!--
 
         <category name="jclouds.wire"> <priority value="DEBUG" />