Merge pull request #409 from dralves/jclouds-vbox-0conf

0conf for vbox. config dirs are created, isos downloaded and a default yaml file is used

labs/virtualbox/src/main/java/org/jclouds/virtualbox/VirtualBoxPropertiesBuilder.java

@@ -73,18 +73,14 @@ public class VirtualBoxPropertiesBuilder extends PropertiesBuilder {
                + "keyboard-configuration/layout=USA keyboard-configuration/variant=USA console-setup/ask_detect=false "
                + "initrd=/install/initrd.gz -- <Enter>");
 
-      properties.put(
+      String workingDir = System.getProperty("test.virtualbox.workingDir", VIRTUALBOX_DEFAULT_DIR);
-               VIRTUALBOX_WORKINGDIR,
-               System.getProperty("user.home") + File.separator
-                        + System.getProperty("test.virtualbox.workingDir", ".jclouds-vbox"));
 
-      // allow to set the descriptor as a sysprop but default to just setting a default file path.
+      properties.put(VIRTUALBOX_WORKINGDIR, workingDir);
-      // The configured supplier
+
-      // must be able to handle the chosen option.
+      String yamlDescriptor = System.getProperty("test.virtualbox.image.descriptor.yaml", VIRTUALBOX_WORKINGDIR
-      properties.put(
+               + File.separator + "images.yaml");
-               VIRTUALBOX_IMAGES_DESCRIPTOR,
+
-               System.getProperty("test.virtualbox.image.descriptor.yaml", VIRTUALBOX_DEFAULT_DIR + File.separator
+      properties.put(VIRTUALBOX_IMAGES_DESCRIPTOR, yamlDescriptor);
-                        + "images.yaml"));
 
       properties.put(VIRTUALBOX_PRECONFIGURATION_URL, "http://10.0.2.2:8080/src/test/resources/preseed.cfg");
 

labs/virtualbox/src/main/java/org/jclouds/virtualbox/config/VirtualBoxComputeServiceContextModule.java

@@ -21,6 +21,7 @@ package org.jclouds.virtualbox.config;
 
 import static org.jclouds.virtualbox.config.VirtualBoxConstants.VIRTUALBOX_PRECONFIGURATION_URL;
 
+import java.io.File;
 import java.io.InputStream;
 import java.net.URI;
 import java.util.Collections;
@@ -73,6 +74,7 @@ import org.jclouds.virtualbox.functions.IMachineToSshClient;
 import org.jclouds.virtualbox.functions.MastersCache;
 import org.jclouds.virtualbox.functions.NodeCreator;
 import org.jclouds.virtualbox.functions.YamlImagesFromFileConfig;
+import org.jclouds.virtualbox.functions.admin.FileDownloadFromURI;
 import org.jclouds.virtualbox.functions.admin.ImagesToYamlImagesFromYamlDescriptor;
 import org.jclouds.virtualbox.functions.admin.StartJettyIfNotAlreadyRunning;
 import org.jclouds.virtualbox.functions.admin.StartVBoxIfNotAlreadyRunning;
@@ -126,6 +128,9 @@ public class VirtualBoxComputeServiceContextModule extends
       }).to(IMachineToImage.class);
       bind(new TypeLiteral<CacheLoader<IsoSpec, URI>>() {
       }).to((Class) StartJettyIfNotAlreadyRunning.class);
+      bind(new TypeLiteral<Function<URI, File>>() {
+      }).to((Class) FileDownloadFromURI.class);
+      
       bind(new TypeLiteral<Supplier<VirtualBoxManager>>() {
       }).to((Class) StartVBoxIfNotAlreadyRunning.class);
       // the yaml config to image mapper

labs/virtualbox/src/main/java/org/jclouds/virtualbox/functions/MastersCache.java

@@ -28,15 +28,20 @@ import static org.jclouds.virtualbox.config.VirtualBoxConstants.VIRTUALBOX_WORKI
 import static org.jclouds.virtualbox.util.MachineUtils.machineNotFoundException;
 
 import java.io.File;
+import java.net.URI;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.concurrent.ExecutionException;
 
+import javax.annotation.PostConstruct;
+import javax.annotation.Resource;
 import javax.inject.Inject;
 import javax.inject.Named;
 
 import org.jclouds.Constants;
 import org.jclouds.compute.domain.Image;
+import org.jclouds.compute.reference.ComputeServiceConstants;
+import org.jclouds.logging.Logger;
 import org.jclouds.virtualbox.domain.HardDisk;
 import org.jclouds.virtualbox.domain.IsoSpec;
 import org.jclouds.virtualbox.domain.Master;
@@ -69,20 +74,26 @@ import com.google.common.collect.Maps;
  */
 public class MastersCache extends AbstractLoadingCache<Image, Master> {
 
+   @Resource
+   @Named(ComputeServiceConstants.COMPUTE_LOGGER)
+   protected Logger logger = Logger.NULL;
+
    private final Map<String, Master> masters = Maps.newHashMap();
    private final Function<MasterSpec, IMachine> masterCreatorAndInstaller;
    private final Map<String, YamlImage> imageMapping;
    private final String workingDir;
-   private final String guestAdditionsIso;
    private final String installationKeySequence;
    private final String isosDir;
    private Supplier<VirtualBoxManager> manager;
+   private Function<URI, File> isoDownloader;
+   private String version;
 
    @Inject
    public MastersCache(@Named(Constants.PROPERTY_BUILD_VERSION) String version,
             @Named(VIRTUALBOX_INSTALLATION_KEY_SEQUENCE) String installationKeySequence,
             @Named(VIRTUALBOX_WORKINGDIR) String workingDir, Function<MasterSpec, IMachine> masterLoader,
-            Supplier<Map<Image, YamlImage>> yamlMapper, Supplier<VirtualBoxManager> manager) {
+            Supplier<Map<Image, YamlImage>> yamlMapper, Supplier<VirtualBoxManager> manager,
+            Function<URI, File> isoDownloader) {
       checkNotNull(version, "version");
       checkNotNull(installationKeySequence, "installationKeySequence");
       checkNotNull(manager, "vboxmanager");
@@ -90,27 +101,37 @@ public class MastersCache extends AbstractLoadingCache<Image, Master> {
       this.masterCreatorAndInstaller = masterLoader;
       this.installationKeySequence = installationKeySequence;
       this.workingDir = workingDir == null ? VIRTUALBOX_DEFAULT_DIR : workingDir;
-      File wdFile = new File(this.workingDir);
+      this.isosDir = workingDir + File.separator + "isos";
-      if (!wdFile.exists()) {
-         wdFile.mkdirs();
-      }
-      this.isosDir = wdFile.getAbsolutePath() + File.separator + "isos";
       this.imageMapping = Maps.newLinkedHashMap();
       for (Entry<Image, YamlImage> entry : yamlMapper.get().entrySet()) {
          this.imageMapping.put(entry.getKey().getId(), entry.getValue());
       }
-      this.guestAdditionsIso = String.format("%s/VBoxGuestAdditions_%s.iso", isosDir,
+      this.version = Iterables.get(Splitter.on('r').split(version), 0);
-               Iterables.get(Splitter.on('r').split(version), 0));
+      this.isoDownloader = isoDownloader;
-      checkState(new File(guestAdditionsIso).exists(), "guest additions iso does not exist at: " + guestAdditionsIso);
+   }
+   
+   @PostConstruct
+   public void createCacheDirStructure() {
+      if (!new File(workingDir).exists()) {
+         new File(workingDir, "isos").mkdirs();
+      }
    }
 
    @Override
-   public Master get(Image key) throws ExecutionException {
+   public synchronized Master get(Image key) throws ExecutionException {
       // check if we have loaded this machine before
       if (masters.containsKey(key.getId())) {
          return masters.get(key);
       }
 
+      String guestAdditionsFileName = String.format("VBoxGuestAdditions_%s.iso", version);
+      String guestAdditionsIso = String.format("%s/%s", isosDir, guestAdditionsFileName);
+      String guestAdditionsUri = "http://download.virtualbox.org/virtualbox/" + version + "/" + guestAdditionsFileName;
+      if (!new File(guestAdditionsIso).exists()) {
+         getFilePathOrDownload(guestAdditionsUri);
+      }
+      checkState(new File(guestAdditionsIso).exists(), "guest additions iso does not exist at: " + guestAdditionsIso);
+
       // the yaml image
       YamlImage yamlImage = imageMapping.get(key.getId());
 
@@ -169,10 +190,12 @@ public class MastersCache extends AbstractLoadingCache<Image, Master> {
    }
 
    private String getFilePathOrDownload(String httpUrl) throws ExecutionException {
-      // TODO validation
       String fileName = httpUrl.substring(httpUrl.lastIndexOf('/') + 1, httpUrl.length());
       File localFile = new File(isosDir, fileName);
-      // TODO download. for now just expect the file to be there
+      if (!localFile.exists()) {
+         logger.debug("iso not found in cache, downloading: %s", httpUrl);
+         localFile = isoDownloader.apply(URI.create(httpUrl));
+      }
       checkState(localFile.exists(), "iso file has not been downloaded: " + fileName);
       return localFile.getAbsolutePath();
    }

labs/virtualbox/src/main/java/org/jclouds/virtualbox/functions/YamlImagesFromFileConfig.java

@@ -20,11 +20,11 @@
 package org.jclouds.virtualbox.functions;
 
 import static com.google.common.base.Preconditions.checkNotNull;
-import static com.google.common.base.Preconditions.checkState;
 
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
+import java.io.InputStreamReader;
 
 import javax.inject.Inject;
 import javax.inject.Named;
@@ -52,11 +52,17 @@ public class YamlImagesFromFileConfig implements Supplier<String> {
 
    @Override
    public String get() {
-      checkNotNull(yamlFilePath, "yaml file path");
-      File yamlFile = new File(yamlFilePath);
-      checkState(yamlFile.exists(), "yaml file does not exist at: " + yamlFilePath);
       try {
-         return IOUtils.toString(new FileInputStream(yamlFile));
+         File yamlFile = new File(yamlFilePath);
+         String yamlDesc = null;
+         // if the yaml file does not exist just use default-images.yaml
+         if (!yamlFile.exists()) {
+            yamlDesc = IOUtils.toString(new InputStreamReader(getClass().getResourceAsStream("/default-images.yaml")));
+         } else {
+            yamlDesc = IOUtils.toString(new FileInputStream(yamlFile));
+         }
+         checkNotNull(yamlDesc, "yaml descriptor");
+         return yamlDesc;
       } catch (IOException e) {
          throw new RuntimeException("error reading yaml file");
       }

labs/virtualbox/src/main/java/org/jclouds/virtualbox/functions/admin/FileDownloadFromURI.java

@@ -38,9 +38,10 @@ import javax.inject.Named;
 import org.jclouds.compute.reference.ComputeServiceConstants;
 import org.jclouds.javax.annotation.Nullable;
 import org.jclouds.logging.Logger;
-import org.jclouds.rest.HttpClient;
+import org.jclouds.rest.HttpAsyncClient;
 
 import com.google.common.base.Function;
+import com.google.common.base.Throwables;
 
 /**
  * @author Mattias Holmqvist
@@ -51,37 +52,40 @@ public class FileDownloadFromURI implements Function<URI, File> {
    @Named(ComputeServiceConstants.COMPUTE_LOGGER)
    protected Logger logger = Logger.NULL;
 
-   private final HttpClient client;
+   private final HttpAsyncClient client;
-   private final String workingDir;
+   private final String isosDir;
 
    @Inject
-   public FileDownloadFromURI(HttpClient client, @Named(VIRTUALBOX_WORKINGDIR) String workingDir) {
+   public FileDownloadFromURI(HttpAsyncClient client, @Named(VIRTUALBOX_WORKINGDIR) String workingDir) {
       this.client = client;
-      this.workingDir = workingDir;
+      this.isosDir = workingDir + File.separator + "isos";
    }
 
    @Override
    public File apply(@Nullable URI input) {
-
+      final File file = new File(isosDir, new File(input.getPath()).getName());
-      final File file = new File(workingDir, new File(input.getPath()).getName());
+      try {
-
+         if (!file.exists()) {
-      if (!file.exists()) {
+            final InputStream inputStream = client.get(input).get();
-         final InputStream inputStream = client.get(input);
+            checkNotNull(inputStream, "%s not found", input);
-         checkNotNull(inputStream, "%s not found", input);
+            try {
-         try {
+               copy(inputStream, new FileOutputStream(file));
-            copy(inputStream, new FileOutputStream(file));
+               return file;
+            } catch (FileNotFoundException e) {
+               logger.error(e, "File %s could not be found", file.getPath());
+            } catch (IOException e) {
+               logger.error(e, "Error when downloading file %s", input.toString());
+            } finally {
+               closeQuietly(inputStream);
+            }
+            return null;
+         } else {
+            logger.debug("File %s already exists. Skipping download", file.getPath());
             return file;
-         } catch (FileNotFoundException e) {
-            logger.error(e, "File %s could not be found", file);
-         } catch (IOException e) {
-            logger.error(e, "Error when downloading file %s", input);
-         } finally {
-            closeQuietly(inputStream);
          }
+      } catch (Exception e) {
+         Throwables.propagate(e);
          return null;
-      } else {
-         logger.debug("File %s already exists. Skipping download", file);
-         return file;
       }
    }
 }

labs/virtualbox/src/main/resources/default-images.yaml

@@ -0,0 +1,68 @@
+images:
+    - id: default-ubuntu-11.04-i386
+      name: ubuntu-11.04-server-i386
+      description: ubuntu 11.04 server (i386)
+      os_arch: x86
+      os_family: ubuntu
+      os_description: ubuntu
+      os_version: 11.04
+      iso: http://releases.ubuntu.com/11.04/ubuntu-11.04-server-i386.iso
+      keystroke_sequence: |
+                <Esc><Esc><Enter> 
+                /install/vmlinuz noapic preseed/url=http://10.0.2.2:8080/src/test/resources/preseed.cfg 
+                debian-installer=en_US auto locale=en_US kbd-chooser/method=us 
+                hostname=vmName 
+                fb=false debconf/frontend=noninteractive 
+                keyboard-configuration/layout=USA keyboard-configuration/variant=USA console-setup/ask_detect=false 
+                initrd=/install/initrd.gz -- <Enter>
+      preseed_cfg: |                
+                      ## Options to set on the command line
+                      d-i debian-installer/locale string en_US.utf8
+                      d-i console-setup/ask_detect boolean false
+                      d-i console-setup/layout string USA
+                      d-i netcfg/get_hostname string unassigned-hostname
+                      d-i netcfg/get_domain string unassigned-domain
+                      # Continue without a default route
+                      # Not working , specify a dummy in the DHCP
+                      d-i time/zone string UTC
+                      d-i clock-setup/utc-auto boolean true
+                      d-i clock-setup/utc boolean true
+                      d-i kbd-chooser/method	select	American English
+                      d-i netcfg/wireless_wep string
+                      d-i base-installer/kernel/override-image string linux-server
+                      # Choices: Dialog, Readline, Gnome, Kde, Editor, Noninteractive
+                      d-i debconf debconf/frontend select Noninteractive
+                      d-i pkgsel/install-language-support boolean false
+                      tasksel tasksel/first multiselect standard, ubuntu-server
+                      d-i partman-auto/method string lvm
+                      d-i partman-lvm/confirm boolean true
+                      d-i partman-lvm/device_remove_lvm boolean true
+                      d-i partman-auto/choose_recipe select atomic
+                      d-i partman/confirm_write_new_label boolean true
+                      d-i partman/confirm_nooverwrite boolean true
+                      d-i partman/choose_partition select finish
+                      d-i partman/confirm boolean true
+                      # Write the changes to disks and configure LVM?
+                      d-i partman-lvm/confirm boolean true
+                      d-i partman-lvm/confirm_nooverwrite boolean true
+                      d-i partman-auto-lvm/guided_size string max
+                      ## Default user, we can get away with a recipe to change this
+                      d-i passwd/user-fullname string toor
+                      d-i passwd/username string toor
+                      d-i passwd/user-password password password
+                      d-i passwd/user-password-again password password
+                      d-i user-setup/encrypt-home boolean false
+                      d-i user-setup/allow-password-weak boolean true
+                      # Individual additional packages to install
+                      d-i pkgsel/include string openssh-server ntp
+                      # Whether to upgrade packages after debootstrap.
+                      # Allowed values: none, safe-upgrade, full-upgrade
+                      d-i pkgsel/upgrade select full-upgrade
+                      d-i grub-installer/only_debian boolean true
+                      d-i grub-installer/with_other_os boolean true
+                      d-i finish-install/reboot_in_progress note
+                      #For the update
+                      d-i pkgsel/update-policy select none
+                      # debconf-get-selections --install
+                      #Use mirror
+                      choose-mirror-bin mirror/http/proxy string

labs/virtualbox/src/test/java/org/jclouds/virtualbox/BaseVirtualBoxClientLiveTest.java

@@ -138,10 +138,7 @@ public class BaseVirtualBoxClientLiveTest extends BaseVersionedServiceLiveTest {
       
       imageId = "ubuntu-11.04-server-i386";
       isosDir = workingDir + File.separator + "isos";
-      File isosDirFile = new File(isosDir);
+      
-      if (!isosDirFile.exists()) {
-         isosDirFile.mkdirs();
-      }
       hostVersion = Iterables.get(Splitter.on('r').split(context.getProviderSpecificContext().getBuildVersion()), 0);
       operatingSystemIso = String.format("%s/%s.iso", isosDir, imageId);
       guestAdditionsIso = String.format("%s/VBoxGuestAdditions_%s.iso", isosDir, hostVersion);

labs/virtualbox/src/test/java/org/jclouds/virtualbox/compute/VirtualBoxComputeServiceAdapterLiveTest.java

@@ -32,6 +32,7 @@ import org.jclouds.domain.LoginCredentials;
 import org.jclouds.ssh.SshClient;
 import org.jclouds.virtualbox.BaseVirtualBoxClientLiveTest;
 import org.jclouds.virtualbox.functions.IMachineToSshClient;
+import org.testng.annotations.AfterClass;
 import org.testng.annotations.Test;
 import org.virtualbox_4_1.IMachine;
 
@@ -48,7 +49,7 @@ public class VirtualBoxComputeServiceAdapterLiveTest extends BaseVirtualBoxClien
    public void testCreatedNodeHasExpectedNameAndWeCanConnectViaSsh() {
       String group = "foo";
       String name = "foo-ef4";
-      String machineName = VIRTUALBOX_NODE_PREFIX + "myTestId-" + group + "-" + name;
+      String machineName = VIRTUALBOX_NODE_PREFIX + "default-ubuntu-11.04-i386-" + group + "-" + name;
 
       Template template = context.getComputeService().templateBuilder().build();
       machine = adapter.createNodeWithGroupEncodedIntoName(group, name, template);
@@ -57,21 +58,6 @@ public class VirtualBoxComputeServiceAdapterLiveTest extends BaseVirtualBoxClien
       doConnectViaSsh(machine.getNode(), prioritizeCredentialsFromTemplate.apply(template, machine.getCredentials()));
    }
 
-   protected void doConnectViaSsh(IMachine machine, LoginCredentials creds) {
-      SshClient ssh = context.utils().injector().getInstance(IMachineToSshClient.class).apply(machine);
-      try {
-         ssh.connect();
-         ExecResponse hello = ssh.exec("echo hello");
-         assertEquals(hello.getOutput().trim(), "hello");
-         System.err.println(ssh.exec("df -k").getOutput());
-         System.err.println(ssh.exec("mount").getOutput());
-         System.err.println(ssh.exec("uname -a").getOutput());
-      } finally {
-         if (ssh != null)
-            ssh.disconnect();
-      }
-   }
-
    @Test
    public void testListHardwareProfiles() {
       Iterable<IMachine> profiles = adapter.listHardwareProfiles();
@@ -85,7 +71,23 @@ public class VirtualBoxComputeServiceAdapterLiveTest extends BaseVirtualBoxClien
       assertEquals(1, Iterables.size(iMageIterable));
       //TODO: check state;
    }
+   
+   protected void doConnectViaSsh(IMachine machine, LoginCredentials creds) {
+      SshClient ssh = context.utils().injector().getInstance(IMachineToSshClient.class).apply(machine);
+      try {
+         ssh.connect();
+         ExecResponse hello = ssh.exec("echo hello");
+         assertEquals(hello.getOutput().trim(), "hello");
+         System.err.println(ssh.exec("df -k").getOutput());
+         System.err.println(ssh.exec("mount").getOutput());
+         System.err.println(ssh.exec("uname -a").getOutput());
+      } finally {
+         if (ssh != null)
+            ssh.disconnect();
+      }
+   }
 
+   @AfterClass
    @Override
    protected void tearDown() throws Exception {
       if (machine != null)

labs/virtualbox/src/test/resources/logback.xml

@@ -49,7 +49,7 @@
     </logger>
 
     <logger name="jclouds.wire">
-        <level value="DEBUG" />
+        <level value="INFO" />
         <appender-ref ref="WIREFILE" />
     </logger>