From e915fd3e7b3eef2acdce4a4b388e9be407da0065 Mon Sep 17 00:00:00 2001 From: David Ribeiro Alves Date: Thu, 8 Mar 2012 02:38:30 +0000 Subject: [PATCH] 0conf for vbox. config dirs are created, isos downloaded and a default .yaml file is used --- .../VirtualBoxPropertiesBuilder.java | 22 +++--- ...VirtualBoxComputeServiceContextModule.java | 5 ++ .../virtualbox/functions/MastersCache.java | 41 +++++++---- .../functions/YamlImagesFromFileConfig.java | 16 +++-- .../functions/admin/FileDownloadFromURI.java | 48 +++++++------ .../src/main/resources/default-images.yaml | 68 +++++++++++++++++++ ...rtualBoxComputeServiceAdapterLiveTest.java | 34 +++++----- .../virtualbox/src/test/resources/logback.xml | 2 +- 8 files changed, 168 insertions(+), 68 deletions(-) create mode 100644 labs/virtualbox/src/main/resources/default-images.yaml diff --git a/labs/virtualbox/src/main/java/org/jclouds/virtualbox/VirtualBoxPropertiesBuilder.java b/labs/virtualbox/src/main/java/org/jclouds/virtualbox/VirtualBoxPropertiesBuilder.java index de41849b58..44d8b3a640 100644 --- a/labs/virtualbox/src/main/java/org/jclouds/virtualbox/VirtualBoxPropertiesBuilder.java +++ b/labs/virtualbox/src/main/java/org/jclouds/virtualbox/VirtualBoxPropertiesBuilder.java @@ -73,18 +73,18 @@ public class VirtualBoxPropertiesBuilder extends PropertiesBuilder { + "keyboard-configuration/layout=USA keyboard-configuration/variant=USA console-setup/ask_detect=false " + "initrd=/install/initrd.gz -- "); - properties.put( - VIRTUALBOX_WORKINGDIR, - System.getProperty("user.home") + File.separator - + System.getProperty("test.virtualbox.workingDir", ".jclouds-vbox")); + String workingDir = System.getProperty("test.virtualbox.workingDir", VIRTUALBOX_DEFAULT_DIR); - // allow to set the descriptor as a sysprop but default to just setting a default file path. - // The configured supplier - // must be able to handle the chosen option. - properties.put( - VIRTUALBOX_IMAGES_DESCRIPTOR, - System.getProperty("test.virtualbox.image.descriptor.yaml", VIRTUALBOX_DEFAULT_DIR + File.separator - + "images.yaml")); + properties.put(VIRTUALBOX_WORKINGDIR, workingDir); + + if (!new File(workingDir).exists()) { + new File(workingDir, "isos").mkdirs(); + } + + String yamlDescriptor = System.getProperty("test.virtualbox.image.descriptor.yaml", VIRTUALBOX_WORKINGDIR + + File.separator + "images.yaml"); + + properties.put(VIRTUALBOX_IMAGES_DESCRIPTOR, yamlDescriptor); properties.put(VIRTUALBOX_PRECONFIGURATION_URL, "http://10.0.2.2:8080/src/test/resources/preseed.cfg"); diff --git a/labs/virtualbox/src/main/java/org/jclouds/virtualbox/config/VirtualBoxComputeServiceContextModule.java b/labs/virtualbox/src/main/java/org/jclouds/virtualbox/config/VirtualBoxComputeServiceContextModule.java index 3a22775be7..0afe266959 100644 --- a/labs/virtualbox/src/main/java/org/jclouds/virtualbox/config/VirtualBoxComputeServiceContextModule.java +++ b/labs/virtualbox/src/main/java/org/jclouds/virtualbox/config/VirtualBoxComputeServiceContextModule.java @@ -21,6 +21,7 @@ package org.jclouds.virtualbox.config; import static org.jclouds.virtualbox.config.VirtualBoxConstants.VIRTUALBOX_PRECONFIGURATION_URL; +import java.io.File; import java.io.InputStream; import java.net.URI; import java.util.Collections; @@ -73,6 +74,7 @@ import org.jclouds.virtualbox.functions.IMachineToSshClient; import org.jclouds.virtualbox.functions.MastersCache; import org.jclouds.virtualbox.functions.NodeCreator; import org.jclouds.virtualbox.functions.YamlImagesFromFileConfig; +import org.jclouds.virtualbox.functions.admin.FileDownloadFromURI; import org.jclouds.virtualbox.functions.admin.ImagesToYamlImagesFromYamlDescriptor; import org.jclouds.virtualbox.functions.admin.StartJettyIfNotAlreadyRunning; import org.jclouds.virtualbox.functions.admin.StartVBoxIfNotAlreadyRunning; @@ -126,6 +128,9 @@ public class VirtualBoxComputeServiceContextModule extends }).to(IMachineToImage.class); bind(new TypeLiteral>() { }).to((Class) StartJettyIfNotAlreadyRunning.class); + bind(new TypeLiteral>() { + }).to((Class) FileDownloadFromURI.class); + bind(new TypeLiteral>() { }).to((Class) StartVBoxIfNotAlreadyRunning.class); // the yaml config to image mapper diff --git a/labs/virtualbox/src/main/java/org/jclouds/virtualbox/functions/MastersCache.java b/labs/virtualbox/src/main/java/org/jclouds/virtualbox/functions/MastersCache.java index bd909bdcce..f8f5d7abd9 100644 --- a/labs/virtualbox/src/main/java/org/jclouds/virtualbox/functions/MastersCache.java +++ b/labs/virtualbox/src/main/java/org/jclouds/virtualbox/functions/MastersCache.java @@ -28,15 +28,19 @@ import static org.jclouds.virtualbox.config.VirtualBoxConstants.VIRTUALBOX_WORKI import static org.jclouds.virtualbox.util.MachineUtils.machineNotFoundException; import java.io.File; +import java.net.URI; import java.util.Map; import java.util.Map.Entry; import java.util.concurrent.ExecutionException; +import javax.annotation.Resource; import javax.inject.Inject; import javax.inject.Named; import org.jclouds.Constants; import org.jclouds.compute.domain.Image; +import org.jclouds.compute.reference.ComputeServiceConstants; +import org.jclouds.logging.Logger; import org.jclouds.virtualbox.domain.HardDisk; import org.jclouds.virtualbox.domain.IsoSpec; import org.jclouds.virtualbox.domain.Master; @@ -69,20 +73,26 @@ import com.google.common.collect.Maps; */ public class MastersCache extends AbstractLoadingCache { + @Resource + @Named(ComputeServiceConstants.COMPUTE_LOGGER) + protected Logger logger = Logger.NULL; + private final Map masters = Maps.newHashMap(); private final Function masterCreatorAndInstaller; private final Map imageMapping; private final String workingDir; - private final String guestAdditionsIso; private final String installationKeySequence; private final String isosDir; private Supplier manager; + private Function isoDownloader; + private String version; @Inject public MastersCache(@Named(Constants.PROPERTY_BUILD_VERSION) String version, @Named(VIRTUALBOX_INSTALLATION_KEY_SEQUENCE) String installationKeySequence, @Named(VIRTUALBOX_WORKINGDIR) String workingDir, Function masterLoader, - Supplier> yamlMapper, Supplier manager) { + Supplier> yamlMapper, Supplier manager, + Function isoDownloader) { checkNotNull(version, "version"); checkNotNull(installationKeySequence, "installationKeySequence"); checkNotNull(manager, "vboxmanager"); @@ -90,27 +100,30 @@ public class MastersCache extends AbstractLoadingCache { this.masterCreatorAndInstaller = masterLoader; this.installationKeySequence = installationKeySequence; this.workingDir = workingDir == null ? VIRTUALBOX_DEFAULT_DIR : workingDir; - File wdFile = new File(this.workingDir); - if (!wdFile.exists()) { - wdFile.mkdirs(); - } - this.isosDir = wdFile.getAbsolutePath() + File.separator + "isos"; + this.isosDir = workingDir + File.separator + "isos"; this.imageMapping = Maps.newLinkedHashMap(); for (Entry entry : yamlMapper.get().entrySet()) { this.imageMapping.put(entry.getKey().getId(), entry.getValue()); } - this.guestAdditionsIso = String.format("%s/VBoxGuestAdditions_%s.iso", isosDir, - Iterables.get(Splitter.on('r').split(version), 0)); - checkState(new File(guestAdditionsIso).exists(), "guest additions iso does not exist at: " + guestAdditionsIso); + this.version = Iterables.get(Splitter.on('r').split(version), 0); + this.isoDownloader = isoDownloader; } @Override - public Master get(Image key) throws ExecutionException { + public synchronized Master get(Image key) throws ExecutionException { // check if we have loaded this machine before if (masters.containsKey(key.getId())) { return masters.get(key); } + String guestAdditionsFileName = String.format("VBoxGuestAdditions_%s.iso", version); + String guestAdditionsIso = String.format("%s/%s", isosDir, guestAdditionsFileName); + String guestAdditionsUri = "http://download.virtualbox.org/virtualbox/" + version + "/" + guestAdditionsFileName; + if (!new File(guestAdditionsIso).exists()) { + getFilePathOrDownload(guestAdditionsUri); + } + checkState(new File(guestAdditionsIso).exists(), "guest additions iso does not exist at: " + guestAdditionsIso); + // the yaml image YamlImage yamlImage = imageMapping.get(key.getId()); @@ -169,10 +182,12 @@ public class MastersCache extends AbstractLoadingCache { } private String getFilePathOrDownload(String httpUrl) throws ExecutionException { - // TODO validation String fileName = httpUrl.substring(httpUrl.lastIndexOf('/') + 1, httpUrl.length()); File localFile = new File(isosDir, fileName); - // TODO download. for now just expect the file to be there + if (!localFile.exists()) { + logger.debug("iso not found in cache, downloading: %s", httpUrl); + localFile = isoDownloader.apply(URI.create(httpUrl)); + } checkState(localFile.exists(), "iso file has not been downloaded: " + fileName); return localFile.getAbsolutePath(); } diff --git a/labs/virtualbox/src/main/java/org/jclouds/virtualbox/functions/YamlImagesFromFileConfig.java b/labs/virtualbox/src/main/java/org/jclouds/virtualbox/functions/YamlImagesFromFileConfig.java index fd772274c9..63a3a2a3d3 100644 --- a/labs/virtualbox/src/main/java/org/jclouds/virtualbox/functions/YamlImagesFromFileConfig.java +++ b/labs/virtualbox/src/main/java/org/jclouds/virtualbox/functions/YamlImagesFromFileConfig.java @@ -20,11 +20,11 @@ package org.jclouds.virtualbox.functions; import static com.google.common.base.Preconditions.checkNotNull; -import static com.google.common.base.Preconditions.checkState; import java.io.File; import java.io.FileInputStream; import java.io.IOException; +import java.io.InputStreamReader; import javax.inject.Inject; import javax.inject.Named; @@ -52,11 +52,17 @@ public class YamlImagesFromFileConfig implements Supplier { @Override public String get() { - checkNotNull(yamlFilePath, "yaml file path"); - File yamlFile = new File(yamlFilePath); - checkState(yamlFile.exists(), "yaml file does not exist at: " + yamlFilePath); try { - return IOUtils.toString(new FileInputStream(yamlFile)); + File yamlFile = new File(yamlFilePath); + String yamlDesc = null; + // if the yaml file does not exist just use default-images.yaml + if (!yamlFile.exists()) { + yamlDesc = IOUtils.toString(new InputStreamReader(getClass().getResourceAsStream("/default-images.yaml"))); + } else { + yamlDesc = IOUtils.toString(new FileInputStream(yamlFile)); + } + checkNotNull(yamlDesc, "yaml descriptor"); + return yamlDesc; } catch (IOException e) { throw new RuntimeException("error reading yaml file"); } diff --git a/labs/virtualbox/src/main/java/org/jclouds/virtualbox/functions/admin/FileDownloadFromURI.java b/labs/virtualbox/src/main/java/org/jclouds/virtualbox/functions/admin/FileDownloadFromURI.java index ccf5cc1b83..013da783dd 100644 --- a/labs/virtualbox/src/main/java/org/jclouds/virtualbox/functions/admin/FileDownloadFromURI.java +++ b/labs/virtualbox/src/main/java/org/jclouds/virtualbox/functions/admin/FileDownloadFromURI.java @@ -38,9 +38,10 @@ import javax.inject.Named; import org.jclouds.compute.reference.ComputeServiceConstants; import org.jclouds.javax.annotation.Nullable; import org.jclouds.logging.Logger; -import org.jclouds.rest.HttpClient; +import org.jclouds.rest.HttpAsyncClient; import com.google.common.base.Function; +import com.google.common.base.Throwables; /** * @author Mattias Holmqvist @@ -51,37 +52,40 @@ public class FileDownloadFromURI implements Function { @Named(ComputeServiceConstants.COMPUTE_LOGGER) protected Logger logger = Logger.NULL; - private final HttpClient client; - private final String workingDir; + private final HttpAsyncClient client; + private final String isosDir; @Inject - public FileDownloadFromURI(HttpClient client, @Named(VIRTUALBOX_WORKINGDIR) String workingDir) { + public FileDownloadFromURI(HttpAsyncClient client, @Named(VIRTUALBOX_WORKINGDIR) String workingDir) { this.client = client; - this.workingDir = workingDir; + this.isosDir = workingDir + File.separator + "isos"; } @Override public File apply(@Nullable URI input) { - - final File file = new File(workingDir, new File(input.getPath()).getName()); - - if (!file.exists()) { - final InputStream inputStream = client.get(input); - checkNotNull(inputStream, "%s not found", input); - try { - copy(inputStream, new FileOutputStream(file)); + final File file = new File(isosDir, new File(input.getPath()).getName()); + try { + if (!file.exists()) { + final InputStream inputStream = client.get(input).get(); + checkNotNull(inputStream, "%s not found", input); + try { + copy(inputStream, new FileOutputStream(file)); + return file; + } catch (FileNotFoundException e) { + logger.error(e, "File %s could not be found", file.getPath()); + } catch (IOException e) { + logger.error(e, "Error when downloading file %s", input.toString()); + } finally { + closeQuietly(inputStream); + } + return null; + } else { + logger.debug("File %s already exists. Skipping download", file.getPath()); return file; - } catch (FileNotFoundException e) { - logger.error(e, "File %s could not be found", file); - } catch (IOException e) { - logger.error(e, "Error when downloading file %s", input); - } finally { - closeQuietly(inputStream); } + } catch (Exception e) { + Throwables.propagate(e); return null; - } else { - logger.debug("File %s already exists. Skipping download", file); - return file; } } } diff --git a/labs/virtualbox/src/main/resources/default-images.yaml b/labs/virtualbox/src/main/resources/default-images.yaml new file mode 100644 index 0000000000..7927b93aac --- /dev/null +++ b/labs/virtualbox/src/main/resources/default-images.yaml @@ -0,0 +1,68 @@ +images: + - id: default-ubuntu-11.04-i386 + name: ubuntu-11.04-server-i386 + description: ubuntu 11.04 server (i386) + os_arch: x86 + os_family: ubuntu + os_description: ubuntu + os_version: 11.04 + iso: http://releases.ubuntu.com/11.04/ubuntu-11.04-server-i386.iso + keystroke_sequence: | + + /install/vmlinuz noapic preseed/url=http://10.0.2.2:8080/src/test/resources/preseed.cfg + debian-installer=en_US auto locale=en_US kbd-chooser/method=us + hostname=vmName + fb=false debconf/frontend=noninteractive + keyboard-configuration/layout=USA keyboard-configuration/variant=USA console-setup/ask_detect=false + initrd=/install/initrd.gz -- + preseed_cfg: | + ## Options to set on the command line + d-i debian-installer/locale string en_US.utf8 + d-i console-setup/ask_detect boolean false + d-i console-setup/layout string USA + d-i netcfg/get_hostname string unassigned-hostname + d-i netcfg/get_domain string unassigned-domain + # Continue without a default route + # Not working , specify a dummy in the DHCP + d-i time/zone string UTC + d-i clock-setup/utc-auto boolean true + d-i clock-setup/utc boolean true + d-i kbd-chooser/method select American English + d-i netcfg/wireless_wep string + d-i base-installer/kernel/override-image string linux-server + # Choices: Dialog, Readline, Gnome, Kde, Editor, Noninteractive + d-i debconf debconf/frontend select Noninteractive + d-i pkgsel/install-language-support boolean false + tasksel tasksel/first multiselect standard, ubuntu-server + d-i partman-auto/method string lvm + d-i partman-lvm/confirm boolean true + d-i partman-lvm/device_remove_lvm boolean true + d-i partman-auto/choose_recipe select atomic + d-i partman/confirm_write_new_label boolean true + d-i partman/confirm_nooverwrite boolean true + d-i partman/choose_partition select finish + d-i partman/confirm boolean true + # Write the changes to disks and configure LVM? + d-i partman-lvm/confirm boolean true + d-i partman-lvm/confirm_nooverwrite boolean true + d-i partman-auto-lvm/guided_size string max + ## Default user, we can get away with a recipe to change this + d-i passwd/user-fullname string toor + d-i passwd/username string toor + d-i passwd/user-password password password + d-i passwd/user-password-again password password + d-i user-setup/encrypt-home boolean false + d-i user-setup/allow-password-weak boolean true + # Individual additional packages to install + d-i pkgsel/include string openssh-server ntp + # Whether to upgrade packages after debootstrap. + # Allowed values: none, safe-upgrade, full-upgrade + d-i pkgsel/upgrade select full-upgrade + d-i grub-installer/only_debian boolean true + d-i grub-installer/with_other_os boolean true + d-i finish-install/reboot_in_progress note + #For the update + d-i pkgsel/update-policy select none + # debconf-get-selections --install + #Use mirror + choose-mirror-bin mirror/http/proxy string diff --git a/labs/virtualbox/src/test/java/org/jclouds/virtualbox/compute/VirtualBoxComputeServiceAdapterLiveTest.java b/labs/virtualbox/src/test/java/org/jclouds/virtualbox/compute/VirtualBoxComputeServiceAdapterLiveTest.java index 0be2cf2d46..82b94fea31 100644 --- a/labs/virtualbox/src/test/java/org/jclouds/virtualbox/compute/VirtualBoxComputeServiceAdapterLiveTest.java +++ b/labs/virtualbox/src/test/java/org/jclouds/virtualbox/compute/VirtualBoxComputeServiceAdapterLiveTest.java @@ -32,6 +32,7 @@ import org.jclouds.domain.LoginCredentials; import org.jclouds.ssh.SshClient; import org.jclouds.virtualbox.BaseVirtualBoxClientLiveTest; import org.jclouds.virtualbox.functions.IMachineToSshClient; +import org.testng.annotations.AfterClass; import org.testng.annotations.Test; import org.virtualbox_4_1.IMachine; @@ -48,7 +49,7 @@ public class VirtualBoxComputeServiceAdapterLiveTest extends BaseVirtualBoxClien public void testCreatedNodeHasExpectedNameAndWeCanConnectViaSsh() { String group = "foo"; String name = "foo-ef4"; - String machineName = VIRTUALBOX_NODE_PREFIX + "myTestId-" + group + "-" + name; + String machineName = VIRTUALBOX_NODE_PREFIX + "default-ubuntu-11.04-i386-" + group + "-" + name; Template template = context.getComputeService().templateBuilder().build(); machine = adapter.createNodeWithGroupEncodedIntoName(group, name, template); @@ -57,21 +58,6 @@ public class VirtualBoxComputeServiceAdapterLiveTest extends BaseVirtualBoxClien doConnectViaSsh(machine.getNode(), prioritizeCredentialsFromTemplate.apply(template, machine.getCredentials())); } - protected void doConnectViaSsh(IMachine machine, LoginCredentials creds) { - SshClient ssh = context.utils().injector().getInstance(IMachineToSshClient.class).apply(machine); - try { - ssh.connect(); - ExecResponse hello = ssh.exec("echo hello"); - assertEquals(hello.getOutput().trim(), "hello"); - System.err.println(ssh.exec("df -k").getOutput()); - System.err.println(ssh.exec("mount").getOutput()); - System.err.println(ssh.exec("uname -a").getOutput()); - } finally { - if (ssh != null) - ssh.disconnect(); - } - } - @Test public void testListHardwareProfiles() { Iterable profiles = adapter.listHardwareProfiles(); @@ -85,7 +71,23 @@ public class VirtualBoxComputeServiceAdapterLiveTest extends BaseVirtualBoxClien assertEquals(1, Iterables.size(iMageIterable)); //TODO: check state; } + + protected void doConnectViaSsh(IMachine machine, LoginCredentials creds) { + SshClient ssh = context.utils().injector().getInstance(IMachineToSshClient.class).apply(machine); + try { + ssh.connect(); + ExecResponse hello = ssh.exec("echo hello"); + assertEquals(hello.getOutput().trim(), "hello"); + System.err.println(ssh.exec("df -k").getOutput()); + System.err.println(ssh.exec("mount").getOutput()); + System.err.println(ssh.exec("uname -a").getOutput()); + } finally { + if (ssh != null) + ssh.disconnect(); + } + } + @AfterClass @Override protected void tearDown() throws Exception { if (machine != null) diff --git a/labs/virtualbox/src/test/resources/logback.xml b/labs/virtualbox/src/test/resources/logback.xml index 4765287d3f..f9b64ed856 100644 --- a/labs/virtualbox/src/test/resources/logback.xml +++ b/labs/virtualbox/src/test/resources/logback.xml @@ -49,7 +49,7 @@ - +