diff --git a/builder/virtualbox/common/step_remove_devices.go b/builder/virtualbox/common/step_remove_devices.go
index 176de50a4..2a11dcbba 100644
--- a/builder/virtualbox/common/step_remove_devices.go
+++ b/builder/virtualbox/common/step_remove_devices.go
@@ -40,19 +40,21 @@ func (s *StepRemoveDevices) Run(state multistep.StateBag) multistep.StepAction {
 		}
 	}
 
-	command := []string{
-		"storageattach", vmName,
-		"--storagectl", "IDE Controller",
-		"--port", "0",
-		"--device", "1",
-		"--medium", "none",
-	}
+	if _, ok := state.GetOk("attachedIso"); ok {
+		command := []string{
+			"storageattach", vmName,
+			"--storagectl", "IDE Controller",
+			"--port", "0",
+			"--device", "1",
+			"--medium", "none",
+		}
 
-	if err := driver.VBoxManage(command...); err != nil {
-		err := fmt.Errorf("Error detaching ISO: %s", err)
-		state.Put("error", err)
-		ui.Error(err.Error())
-		return multistep.ActionHalt
+		if err := driver.VBoxManage(command...); err != nil {
+			err := fmt.Errorf("Error detaching ISO: %s", err)
+			state.Put("error", err)
+			ui.Error(err.Error())
+			return multistep.ActionHalt
+		}
 	}
 
 	return multistep.ActionContinue
diff --git a/builder/virtualbox/common/step_remove_devices_test.go b/builder/virtualbox/common/step_remove_devices_test.go
index af1bccc1f..e74331232 100644
--- a/builder/virtualbox/common/step_remove_devices_test.go
+++ b/builder/virtualbox/common/step_remove_devices_test.go
@@ -25,6 +25,29 @@ func TestStepRemoveDevices(t *testing.T) {
 		t.Fatal("should NOT have error")
 	}
 
+	// Test that ISO was removed
+	if len(driver.VBoxManageCalls) != 0 {
+		t.Fatalf("bad: %#v", driver.VBoxManageCalls)
+	}
+}
+
+func TestStepRemoveDevices_attachedIso(t *testing.T) {
+	state := testState(t)
+	step := new(StepRemoveDevices)
+
+	state.Put("attachedIso", true)
+	state.Put("vmName", "foo")
+
+	driver := state.Get("driver").(*DriverMock)
+
+	// Test the run
+	if action := step.Run(state); action != multistep.ActionContinue {
+		t.Fatalf("bad action: %#v", action)
+	}
+	if _, ok := state.GetOk("error"); ok {
+		t.Fatal("should NOT have error")
+	}
+
 	// Test that ISO was removed
 	if len(driver.VBoxManageCalls) != 1 {
 		t.Fatalf("bad: %#v", driver.VBoxManageCalls)
@@ -52,13 +75,10 @@ func TestStepRemoveDevices_floppyPath(t *testing.T) {
 	}
 
 	// Test that both were removed
-	if len(driver.VBoxManageCalls) != 2 {
+	if len(driver.VBoxManageCalls) != 1 {
 		t.Fatalf("bad: %#v", driver.VBoxManageCalls)
 	}
 	if driver.VBoxManageCalls[0][3] != "Floppy Controller" {
 		t.Fatalf("bad: %#v", driver.VBoxManageCalls)
 	}
-	if driver.VBoxManageCalls[1][3] != "IDE Controller" {
-		t.Fatalf("bad: %#v", driver.VBoxManageCalls)
-	}
 }
diff --git a/builder/virtualbox/iso/step_attach_iso.go b/builder/virtualbox/iso/step_attach_iso.go
index 5d4ec42ef..f18281434 100644
--- a/builder/virtualbox/iso/step_attach_iso.go
+++ b/builder/virtualbox/iso/step_attach_iso.go
@@ -41,6 +41,9 @@ func (s *stepAttachISO) Run(state multistep.StateBag) multistep.StepAction {
 	// Track the path so that we can unregister it from VirtualBox later
 	s.diskPath = isoPath
 
+	// Set some state so we know to remove
+	state.Put("attachedIso", true)
+
 	return multistep.ActionContinue
 }