Merge pull request #443 from dralves/jclouds-vbox-nat-host-only-working-cluster-and-live-test

vbox - last pr broke master creation. worked around qemu bug that prevented clones from adding users. AdminAccess is working

labs/virtualbox/src/main/java/org/jclouds/virtualbox/VirtualBoxPropertiesBuilder.java

@@ -19,18 +19,16 @@
 
 package org.jclouds.virtualbox;
 
-import static org.jclouds.Constants.*;
+import static org.jclouds.Constants.PROPERTY_API_VERSION;
 import static org.jclouds.Constants.PROPERTY_BUILD_VERSION;
-import static org.jclouds.Constants.PROPERTY_CREDENTIAL;
 import static org.jclouds.Constants.PROPERTY_ENDPOINT;
-import static org.jclouds.Constants.PROPERTY_IDENTITY;
 import static org.jclouds.compute.reference.ComputeServiceConstants.PROPERTY_IMAGE_AUTHENTICATE_SUDO;
 import static org.jclouds.compute.reference.ComputeServiceConstants.PROPERTY_IMAGE_LOGIN_USER;
 import static org.jclouds.virtualbox.config.VirtualBoxConstants.VIRTUALBOX_DEFAULT_DIR;
 import static org.jclouds.virtualbox.config.VirtualBoxConstants.VIRTUALBOX_IMAGES_DESCRIPTOR;
 import static org.jclouds.virtualbox.config.VirtualBoxConstants.VIRTUALBOX_INSTALLATION_KEY_SEQUENCE;
 import static org.jclouds.virtualbox.config.VirtualBoxConstants.VIRTUALBOX_PRECONFIGURATION_URL;
-import static org.jclouds.virtualbox.config.VirtualBoxConstants.*;
+import static org.jclouds.virtualbox.config.VirtualBoxConstants.VIRTUALBOX_WORKINGDIR;
 
 import java.io.File;
 import java.util.Properties;
@@ -60,11 +58,9 @@ public class VirtualBoxPropertiesBuilder extends PropertiesBuilder {
       properties.put(PROPERTY_API_VERSION, "4.1.4");
 
       properties.put(PROPERTY_BUILD_VERSION, "4.1.8r75467");
-      properties.put(PROPERTY_IDENTITY, "administrator");
-      properties.put(PROPERTY_CREDENTIAL, "12345");
-
       properties.put(PROPERTY_IMAGE_LOGIN_USER, "toor:password");
       properties.put(PROPERTY_IMAGE_AUTHENTICATE_SUDO, "true");
+      
 
       properties.put(VIRTUALBOX_INSTALLATION_KEY_SEQUENCE, "<Esc><Esc><Enter> "
                + "/install/vmlinuz noapic preseed/url=PRECONFIGURATION_URL "

labs/virtualbox/src/main/java/org/jclouds/virtualbox/functions/CloneAndRegisterMachineFromIMachineIfNotAlreadyExists.java

@@ -24,7 +24,6 @@ import java.util.List;
 
 import javax.annotation.Resource;
 import javax.inject.Named;
-import javax.inject.Singleton;
 
 import org.jclouds.compute.reference.ComputeServiceConstants;
 import org.jclouds.logging.Logger;

labs/virtualbox/src/main/java/org/jclouds/virtualbox/functions/CreateAndInstallVm.java

@@ -18,9 +18,9 @@
  */
 package org.jclouds.virtualbox.functions;
 
+import static org.jclouds.scriptbuilder.domain.Statements.call;
 import static com.google.common.base.Preconditions.checkState;
 import static com.google.common.collect.Iterables.transform;
-import static org.jclouds.scriptbuilder.domain.Statements.call;
 
 import java.net.URI;
 import java.util.List;
@@ -121,18 +121,18 @@ public class CreateAndInstallVm implements Function<MasterSpec, IMachine> {
 				"timed out waiting for guest %s to be accessible via ssh",
 				vmName);
 
-		//logger.debug(">> awaiting installation of guest additions on vm: %s", vmName);
+		logger.debug(">> awaiting installation of guest additions on vm: %s", vmName);
-		//checkState(guestAdditionsInstaller.apply(vm));
+		checkState(guestAdditionsInstaller.apply(vm));
 
 		logger.debug(">> awaiting post-installation actions on vm: %s", vmName);
 
-		NodeMetadata vmMetadata = imachineToNodeMetadata.apply(vm);
+//		NodeMetadata vmMetadata = imachineToNodeMetadata.apply(vm);
 
-		ListenableFuture<ExecResponse> execFuture =
+//		ListenableFuture<ExecResponse> execFuture =
-		machineUtils.runScriptOnNode(vmMetadata, call("cleanupUdevIfNeeded"), RunScriptOptions.NONE);
+//		machineUtils.runScriptOnNode(vmMetadata, call("cleanupUdevIfNeeded"), RunScriptOptions.NONE);
 
-		ExecResponse execResponse = Futures.getUnchecked(execFuture);
+//		ExecResponse execResponse = Futures.getUnchecked(execFuture);
-		checkState(execResponse.getExitStatus() == 0);
+//		checkState(execResponse.getExitStatus() == 0);
 
 		logger.debug(
 				"<< installation of image complete. Powering down node(%s)",

labs/virtualbox/src/main/java/org/jclouds/virtualbox/functions/IMachineToNodeMetadata.java

@@ -23,17 +23,12 @@ import static com.google.common.base.Preconditions.checkNotNull;
 import static com.google.common.base.Preconditions.checkState;
 import static org.jclouds.virtualbox.config.VirtualBoxComputeServiceContextModule.machineToNodeState;
 
-import java.util.HashSet;
-import java.util.Set;
-
 import javax.annotation.Resource;
 import javax.inject.Named;
 
-import org.jclouds.compute.domain.HardwareBuilder;
 import org.jclouds.compute.domain.NodeMetadata;
 import org.jclouds.compute.domain.NodeMetadataBuilder;
 import org.jclouds.compute.domain.NodeState;
-import org.jclouds.compute.domain.Processor;
 import org.jclouds.compute.reference.ComputeServiceConstants;
 import org.jclouds.domain.LocationBuilder;
 import org.jclouds.domain.LocationScope;

labs/virtualbox/src/main/java/org/jclouds/virtualbox/functions/NodeCreator.java

@@ -55,148 +55,124 @@ import com.google.common.base.Function;
 import com.google.common.base.Supplier;
 
 @Singleton
-public class NodeCreator implements
+public class NodeCreator implements Function<NodeSpec, NodeAndInitialCredentials<IMachine>> {
-		Function<NodeSpec, NodeAndInitialCredentials<IMachine>> {
 
-	// TODO parameterize
+   // TODO parameterize
-	public static final int NODE_PORT_INIT = 3000;
+   public static final int NODE_PORT_INIT = 3000;
 
-	// TODO parameterize
+   // TODO parameterize
-	public static final String VMS_NETWORK = "33.33.33.";
+   public static final String VMS_NETWORK = "33.33.33.";
 
-	// TODO parameterize
+   // TODO parameterize
-	public static final String HOST_ONLY_IFACE_NAME = "vboxnet0";
+   public static final String HOST_ONLY_IFACE_NAME = "vboxnet0";
 
-	// TODO parameterize
+   // TODO parameterize
-	public static final boolean USE_LINKED = true;
+   public static final boolean USE_LINKED = true;
 
-	private final Supplier<VirtualBoxManager> manager;
+   private final Supplier<VirtualBoxManager> manager;
-	private final Function<CloneSpec, IMachine> cloner;
+   private final Function<CloneSpec, IMachine> cloner;
-	private final AtomicInteger nodePorts;
+   private final AtomicInteger nodePorts;
-	private final AtomicInteger nodeIps;
+   private final AtomicInteger nodeIps;
-	private MachineUtils machineUtils;
+   private MachineUtils machineUtils;
-	private Function<IMachine, NodeMetadata> imachineToNodeMetadata;
+   private Function<IMachine, NodeMetadata> imachineToNodeMetadata;
 
-	private final RunScriptOnNode.Factory scriptRunnerFactory;
+   private final RunScriptOnNode.Factory scriptRunnerFactory;
-	private final Supplier<NodeMetadata> hostSupplier;
+   private final Supplier<NodeMetadata> hostSupplier;
 
-	@Inject
+   @Inject
-	public NodeCreator(Supplier<VirtualBoxManager> manager,
+   public NodeCreator(Supplier<VirtualBoxManager> manager, Function<CloneSpec, IMachine> cloner,
-			Function<CloneSpec, IMachine> cloner, MachineUtils machineUtils,
+            MachineUtils machineUtils, Function<IMachine, NodeMetadata> imachineToNodeMetadata,
-			Function<IMachine, NodeMetadata> imachineToNodeMetadata,
+            RunScriptOnNode.Factory scriptRunnerFactory, Supplier<NodeMetadata> hostSupplier) {
-			RunScriptOnNode.Factory scriptRunnerFactory,
+      this.manager = manager;
-			Supplier<NodeMetadata> hostSupplier) {
+      this.cloner = cloner;
-		this.manager = manager;
+      this.nodePorts = new AtomicInteger(NODE_PORT_INIT);
-		this.cloner = cloner;
+      this.nodeIps = new AtomicInteger(1);
-		this.nodePorts = new AtomicInteger(NODE_PORT_INIT);
+      this.machineUtils = machineUtils;
-		this.nodeIps = new AtomicInteger(1);
+      this.imachineToNodeMetadata = imachineToNodeMetadata;
-		this.machineUtils = machineUtils;
+      this.scriptRunnerFactory = scriptRunnerFactory;
-		this.imachineToNodeMetadata = imachineToNodeMetadata;
+      this.hostSupplier = hostSupplier;
-		this.scriptRunnerFactory = scriptRunnerFactory;
-		this.hostSupplier = hostSupplier;
 
-	}
+   }
 
-	@Override
+   @Override
-	public NodeAndInitialCredentials<IMachine> apply(NodeSpec nodeSpec) {
+   public NodeAndInitialCredentials<IMachine> apply(NodeSpec nodeSpec) {
 
-		checkNotNull(nodeSpec, "NodeSpec");
+      checkNotNull(nodeSpec, "NodeSpec");
 
-		Master master = nodeSpec.getMaster();
+      Master master = nodeSpec.getMaster();
-		checkNotNull(master, "Master");
+      checkNotNull(master, "Master");
 
-		if (master.getMachine().getCurrentSnapshot() != null) {
+      if (master.getMachine().getCurrentSnapshot() != null) {
-			ISession session;
+         ISession session;
-			try {
+         try {
-				session = manager.get().openMachineSession(master.getMachine());
+            session = manager.get().openMachineSession(master.getMachine());
-			} catch (Exception e) {
+         } catch (Exception e) {
-				throw new RuntimeException(
+            throw new RuntimeException("error opening vbox machine session: " + e.getMessage(), e);
-						"error opening vbox machine session: " + e.getMessage(),
+         }
-						e);
+         session.getConsole().deleteSnapshot(master.getMachine().getCurrentSnapshot().getId());
-			}
+         session.unlockMachine();
-			session.getConsole().deleteSnapshot(
+      }
-					master.getMachine().getCurrentSnapshot().getId());
+      String masterNameWithoutPrefix = master.getSpec().getVmSpec().getVmName().replace(VIRTUALBOX_IMAGE_PREFIX, "");
-			session.unlockMachine();
-		}
-		String masterNameWithoutPrefix = master.getSpec().getVmSpec()
-				.getVmName().replace(VIRTUALBOX_IMAGE_PREFIX, "");
 
-		String cloneName = VIRTUALBOX_NODE_PREFIX + masterNameWithoutPrefix
+      String cloneName = VIRTUALBOX_NODE_PREFIX + masterNameWithoutPrefix + "-" + nodeSpec.getTag() + "-"
-				+ "-" + nodeSpec.getTag() + "-" + nodeSpec.getName();
+               + nodeSpec.getName();
 
-		VmSpec cloneVmSpec = VmSpec.builder().id(cloneName).name(cloneName)
+      VmSpec cloneVmSpec = VmSpec.builder().id(cloneName).name(cloneName).memoryMB(512).cleanUpMode(CleanupMode.Full)
-				.memoryMB(512).cleanUpMode(CleanupMode.Full)
+               .forceOverwrite(true).build();
-				.forceOverwrite(true).build();
 
-		
+      // CASE NAT + HOST-ONLY
-		// CASE NAT + HOST-ONLY 
+      NetworkAdapter natAdapter = NetworkAdapter.builder().networkAttachmentType(NetworkAttachmentType.NAT)
-		NetworkAdapter natAdapter = NetworkAdapter
+               .tcpRedirectRule("127.0.0.1", this.nodePorts.getAndIncrement(), "", 22).build();
-				.builder()
+      NetworkInterfaceCard natIfaceCard = NetworkInterfaceCard.builder().addNetworkAdapter(natAdapter).slot(0L).build();
-				.networkAttachmentType(NetworkAttachmentType.NAT)
-				.tcpRedirectRule("127.0.0.1", this.nodePorts.getAndIncrement(),
-						"", 22).build();
-		NetworkInterfaceCard natIfaceCard = NetworkInterfaceCard.builder()
-				.addNetworkAdapter(natAdapter).slot(0L).build();
 
-		NetworkAdapter hostOnlyAdapter = NetworkAdapter.builder()
+      NetworkAdapter hostOnlyAdapter = NetworkAdapter.builder().networkAttachmentType(NetworkAttachmentType.HostOnly)
-				.networkAttachmentType(NetworkAttachmentType.HostOnly)
+               .staticIp(VMS_NETWORK + this.nodeIps.getAndIncrement()).build();
-				.staticIp(VMS_NETWORK + this.nodeIps.getAndIncrement()).build();
 
-		NetworkInterfaceCard hostOnlyIfaceCard = NetworkInterfaceCard.builder()
+      NetworkInterfaceCard hostOnlyIfaceCard = NetworkInterfaceCard.builder().addNetworkAdapter(hostOnlyAdapter)
-				.addNetworkAdapter(hostOnlyAdapter)
+               .addHostInterfaceName(HOST_ONLY_IFACE_NAME).slot(1L).build();
-				.addHostInterfaceName(HOST_ONLY_IFACE_NAME).slot(1L).build();		
-		
-		NetworkSpec networkSpec = createNetworkSpecForHostOnlyNATNICs(natIfaceCard, hostOnlyIfaceCard);
-		////
-		
-		// CASE BRIDGED
-		//NetworkSpec networkSpec = createNetworkSpecForBridgedNIC();
 
-		CloneSpec cloneSpec = CloneSpec.builder().linked(USE_LINKED)
+      NetworkSpec networkSpec = createNetworkSpecForHostOnlyNATNICs(natIfaceCard, hostOnlyIfaceCard);
-				.master(master.getMachine()).network(networkSpec)
+      // //
-				.vm(cloneVmSpec).build();
 
-		IMachine cloned = cloner.apply(cloneSpec);
+      // CASE BRIDGED
+      // NetworkSpec networkSpec = createNetworkSpecForBridgedNIC();
 
-		new LaunchMachineIfNotAlreadyRunning(manager.get(), ExecutionType.GUI,
+      CloneSpec cloneSpec = CloneSpec.builder().linked(USE_LINKED).master(master.getMachine()).network(networkSpec)
-				"").apply(cloned);
+               .vm(cloneVmSpec).build();
 
-		
+      IMachine cloned = cloner.apply(cloneSpec);
-		 // CASE NAT + HOST-ONLY
-		 machineUtils.runScriptOnNode(imachineToNodeMetadata.apply(cloned),
-		 new SetIpAddress(hostOnlyIfaceCard), RunScriptOptions.NONE);
-		 ////
-		 
 
-		// TODO get credentials from somewhere else (they are also HC in
+      new LaunchMachineIfNotAlreadyRunning(manager.get(), ExecutionType.GUI, "").apply(cloned);
-		// IMachineToSshClient)
-		NodeAndInitialCredentials<IMachine> nodeAndInitialCredentials = new NodeAndInitialCredentials<IMachine>(
-				cloned, cloneName, LoginCredentials.builder().user("toor")
-						.password("password").authenticateSudo(true).build());
 
-		return nodeAndInitialCredentials;
+      // CASE NAT + HOST-ONLY
-	}
+      machineUtils.runScriptOnNode(imachineToNodeMetadata.apply(cloned), new SetIpAddress(hostOnlyIfaceCard),
+               RunScriptOptions.NONE);
+      // //
 
-	private NetworkSpec createNetworkSpecForHostOnlyNATNICs(NetworkInterfaceCard natIfaceCard, NetworkInterfaceCard hostOnlyIfaceCard) {
+      // TODO get credentials from somewhere else (they are also HC in
-		return NetworkSpec.builder().addNIC(natIfaceCard)
+      // IMachineToSshClient)
-				.addNIC(hostOnlyIfaceCard).build();
+      NodeAndInitialCredentials<IMachine> nodeAndInitialCredentials = new NodeAndInitialCredentials<IMachine>(cloned,
-	}
+               cloneName, LoginCredentials.builder().user("toor").password("password").authenticateSudo(true).build());
 
-	private NetworkSpec createNetworkSpecForBridgedNIC() {
+      return nodeAndInitialCredentials;
-		List<BridgedIf> activeBridgedInterfaces = new RetrieveActiveBridgedInterfaces(
+   }
-				scriptRunnerFactory).apply(hostSupplier.get());
-		BridgedIf bridgedActiveInterface = checkNotNull(
-				activeBridgedInterfaces.get(0), "activeBridgedIf");
 
-		NetworkAdapter bridgedAdapter = NetworkAdapter.builder()
+   private NetworkSpec createNetworkSpecForHostOnlyNATNICs(NetworkInterfaceCard natIfaceCard,
-				.networkAttachmentType(NetworkAttachmentType.Bridged).build();
+            NetworkInterfaceCard hostOnlyIfaceCard) {
-		NetworkInterfaceCard bridgedNIC = NetworkInterfaceCard.builder()
+      return NetworkSpec.builder().addNIC(natIfaceCard).addNIC(hostOnlyIfaceCard).build();
-				.addNetworkAdapter(bridgedAdapter)
+   }
-				.addHostInterfaceName(bridgedActiveInterface.getName())
-				.slot(0L).build();
 
-		NetworkSpec networkSpec = NetworkSpec.builder().addNIC(bridgedNIC)
+   private NetworkSpec createNetworkSpecForBridgedNIC() {
-				.build();
+      List<BridgedIf> activeBridgedInterfaces = new RetrieveActiveBridgedInterfaces(scriptRunnerFactory)
-		return networkSpec;
+               .apply(hostSupplier.get());
-	}
+      BridgedIf bridgedActiveInterface = checkNotNull(activeBridgedInterfaces.get(0), "activeBridgedIf");
+
+      NetworkAdapter bridgedAdapter = NetworkAdapter.builder().networkAttachmentType(NetworkAttachmentType.Bridged)
+               .build();
+      NetworkInterfaceCard bridgedNIC = NetworkInterfaceCard.builder().addNetworkAdapter(bridgedAdapter)
+               .addHostInterfaceName(bridgedActiveInterface.getName()).slot(0L).build();
+
+      NetworkSpec networkSpec = NetworkSpec.builder().addNIC(bridgedNIC).build();
+      return networkSpec;
+   }
 }

labs/virtualbox/src/main/java/org/jclouds/virtualbox/functions/admin/StartVBoxIfNotAlreadyRunning.java

@@ -25,6 +25,7 @@ import static org.jclouds.compute.options.RunScriptOptions.Builder.runAsRoot;
 
 import java.net.URI;
 
+import javax.annotation.Nullable;
 import javax.annotation.PostConstruct;
 import javax.annotation.Resource;
 import javax.inject.Inject;
@@ -58,8 +59,8 @@ public class StartVBoxIfNotAlreadyRunning implements Supplier<VirtualBoxManager>
    private final RetryIfSocketNotYetOpen socketTester;
    private final Supplier<NodeMetadata> host;
    private final Supplier<URI> providerSupplier;
-   private final String identity;
+//   private final String identity;
-   private final String credential;
+//   private final String credential;
    private final Function<Supplier<NodeMetadata>, VirtualBoxManager> managerForNode;
    private transient VirtualBoxManager manager;
 
@@ -67,14 +68,15 @@ public class StartVBoxIfNotAlreadyRunning implements Supplier<VirtualBoxManager>
    @Inject
    public StartVBoxIfNotAlreadyRunning(Function<Supplier<NodeMetadata>, VirtualBoxManager> managerForNode,
             Factory runScriptOnNodeFactory, RetryIfSocketNotYetOpen socketTester, Supplier<NodeMetadata> host,
-            @Provider Supplier<URI> providerSupplier, @Identity String identity, @Credential String credential) {
+            @Provider Supplier<URI> providerSupplier, @Nullable @Identity String identity,
+            @Nullable @Credential String credential) {
       this.runScriptOnNodeFactory = checkNotNull(runScriptOnNodeFactory, "runScriptOnNodeFactory");
       this.socketTester = checkNotNull(socketTester, "socketTester");
       this.socketTester.seconds(3L);
       this.host = checkNotNull(host, "host");
       this.providerSupplier = checkNotNull(providerSupplier, "endpoint to virtualbox websrvd is needed");
-      this.identity = checkNotNull(identity, "identity");
+//      this.identity = checkNotNull(identity, "identity");
-      this.credential = checkNotNull(credential, "credential");
+//      this.credential = checkNotNull(credential, "credential");
       this.managerForNode = checkNotNull(managerForNode, "managerForNode");
    }
 
@@ -99,7 +101,7 @@ public class StartVBoxIfNotAlreadyRunning implements Supplier<VirtualBoxManager>
          }
       }
       manager = managerForNode.apply(host);
-      manager.connect(provider.toASCIIString(), identity, credential);
+      manager.connect(provider.toASCIIString(), "", "");
       if (logger.isDebugEnabled())
          if (manager.getSessionObject().getState() != SessionState.Unlocked)
             logger.warn("manager is not in unlocked state " + manager.getSessionObject().getState());

labs/virtualbox/src/main/resources/functions/installModuleAssistantIfNeeded.sh

@@ -10,5 +10,6 @@ function installModuleAssistantIfNeeded {
       mkdir /etc/udev/rules.d/70-persistent-net.rules;
       rm -rf /dev/.udev/;
       rm /lib/udev/rules.d/75-persistent-net-generator.rules
+      rm -f /etc/passwd.lock /etc/group.lock /etc/gshadow.lock
    fi
 }

labs/virtualbox/src/test/java/org/jclouds/virtualbox/compute/VirtualBoxExperimentLiveTest.java

@@ -31,9 +31,11 @@ import org.jclouds.compute.ComputeServiceContextFactory;
 import org.jclouds.compute.RunNodesException;
 import org.jclouds.compute.domain.ExecResponse;
 import org.jclouds.compute.domain.NodeMetadata;
+import org.jclouds.compute.options.TemplateOptions;
 import org.jclouds.compute.reference.ComputeServiceConstants;
 import org.jclouds.logging.Logger;
 import org.jclouds.logging.slf4j.config.SLF4JLoggingModule;
+import org.jclouds.scriptbuilder.statements.login.AdminAccess;
 import org.jclouds.ssh.SshClient;
 import org.jclouds.sshj.config.SshjSshClientModule;
 import org.testng.annotations.BeforeClass;
@@ -58,15 +60,16 @@ public class VirtualBoxExperimentLiveTest {
 
    @BeforeClass
    public void setUp() {
-      context = new ComputeServiceContextFactory().createContext("virtualbox", "toor", "password",
+      context = new ComputeServiceContextFactory().createContext("virtualbox", "", "",
                ImmutableSet.<Module> of(new SLF4JLoggingModule(), new SshjSshClientModule()));
    }
 
    @Test
    public void testLaunchCluster() throws RunNodesException {
-      int numNodes = 4;
+      int numNodes = 1;
       final String clusterName = "test-launch-cluster";
-      Set<? extends NodeMetadata> nodes = context.getComputeService().createNodesInGroup(clusterName, numNodes);
+      Set<? extends NodeMetadata> nodes = context.getComputeService().createNodesInGroup(clusterName, numNodes,
+               TemplateOptions.Builder.runScript(AdminAccess.standard()));
       assertEquals(numNodes, nodes.size(), "wrong number of nodes");
       for (NodeMetadata node : nodes) {
          logger.debug("Created Node: %s", node);