move to using AtomicReference for retryable predicates saves at least one network call on re-fetch

apis/ec2/src/main/java/org/jclouds/ec2/compute/EC2ComputeService.java

@@ -24,10 +24,11 @@ import static com.google.common.collect.Iterables.transform;
 import static org.jclouds.util.Preconditions2.checkNotEmpty;
 
 import java.util.Map;
-import java.util.Set;
 import java.util.Map.Entry;
+import java.util.Set;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.ExecutorService;
+import java.util.concurrent.atomic.AtomicReference;
 
 import javax.inject.Inject;
 import javax.inject.Named;
@@ -71,8 +72,8 @@ import com.google.common.base.Predicate;
 import com.google.common.base.Supplier;
 import com.google.common.cache.LoadingCache;
 import com.google.common.collect.ImmutableMultimap;
-import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.ImmutableMultimap.Builder;
+import com.google.common.collect.ImmutableSet;
 
 /**
  * @author Adrian Cole
@@ -91,9 +92,9 @@ public class EC2ComputeService extends BaseComputeService {
          RebootNodeStrategy rebootNodeStrategy, DestroyNodeStrategy destroyNodeStrategy,
          ResumeNodeStrategy startNodeStrategy, SuspendNodeStrategy stopNodeStrategy,
          Provider<TemplateBuilder> templateBuilderProvider, Provider<TemplateOptions> templateOptionsProvider,
-         @Named("NODE_RUNNING") Predicate<NodeMetadata> nodeRunning,
+         @Named("NODE_RUNNING") Predicate<AtomicReference<NodeMetadata>> nodeRunning,
-         @Named("NODE_TERMINATED") Predicate<NodeMetadata> nodeTerminated,
+         @Named("NODE_TERMINATED") Predicate<AtomicReference<NodeMetadata>> nodeTerminated,
-         @Named("NODE_SUSPENDED") Predicate<NodeMetadata> nodeSuspended,
+         @Named("NODE_SUSPENDED") Predicate<AtomicReference<NodeMetadata>> nodeSuspended,
          InitializeRunScriptOnNodeOrPlaceInBadMap.Factory initScriptRunnerFactory,
          RunScriptOnNode.Factory runScriptOnNodeFactory, InitAdminAccess initAdminAccess,
          PersistNodeCredentials persistNodeCredentials, Timeouts timeouts,

apis/ec2/src/main/java/org/jclouds/ec2/compute/strategy/EC2CreateNodesInGroupThenAddToSet.java

@@ -26,6 +26,7 @@ import static org.jclouds.ec2.compute.util.EC2ComputeUtils.getZoneFromLocationOr
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.Future;
+import java.util.concurrent.atomic.AtomicReference;
 
 import javax.annotation.Resource;
 import javax.inject.Inject;
@@ -80,7 +81,7 @@ public class EC2CreateNodesInGroupThenAddToSet implements CreateNodesInGroupThen
    @VisibleForTesting
    final EC2Client client;
    @VisibleForTesting
-   final Predicate<NodeMetadata> nodeRunning;
+   final Predicate<AtomicReference<NodeMetadata>> nodeRunning;
    @VisibleForTesting
    final LoadingCache<RegionAndName, String> elasticIpCache;
    @VisibleForTesting
@@ -101,7 +102,7 @@ public class EC2CreateNodesInGroupThenAddToSet implements CreateNodesInGroupThen
             EC2Client client,
             @Named("ELASTICIP")
             LoadingCache<RegionAndName, String> elasticIpCache, 
-            @Named("NODE_RUNNING") Predicate<NodeMetadata> nodeRunning,
+            @Named("NODE_RUNNING") Predicate<AtomicReference<NodeMetadata>> nodeRunning,
             Provider<TemplateBuilder> templateBuilderProvider,
             CreateKeyPairAndSecurityGroupsAsNeededAndReturnRunOptions createKeyPairAndSecurityGroupsAsNeededAndReturncustomize,
             InstancePresent instancePresent, Function<RunningInstance, NodeMetadata> runningInstanceToNodeMetadata,
@@ -197,7 +198,8 @@ public class EC2CreateNodesInGroupThenAddToSet implements CreateNodesInGroupThen
 
          // block until instance is running
          logger.debug(">> awaiting status running instance(%s)", coordinates);
-         nodeRunning.apply(runningInstanceToNodeMetadata.apply(startedInstance));
+         AtomicReference<NodeMetadata> node = new AtomicReference<NodeMetadata>(runningInstanceToNodeMetadata.apply(startedInstance));
+         nodeRunning.apply(node);
          logger.trace("<< running instance(%s)", coordinates);
          logger.debug(">> associating elastic IP %s to instance %s", ip, coordinates);
          client.getElasticIPAddressServices().associateAddressInRegion(region, ip, id);

apis/ec2/src/test/java/org/jclouds/ec2/compute/strategy/EC2CreateNodesInGroupThenAddToSetTest.java

@@ -18,12 +18,12 @@
  */
 package org.jclouds.ec2.compute.strategy;
 
+import static org.easymock.EasyMock.createMock;
 import static org.easymock.EasyMock.eq;
 import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.replay;
 import static org.easymock.EasyMock.reportMatcher;
-import static org.easymock.classextension.EasyMock.createMock;
+import static org.easymock.EasyMock.verify;
-import static org.easymock.classextension.EasyMock.replay;
-import static org.easymock.classextension.EasyMock.verify;
 
 import java.util.Map;
 import java.util.Set;
@@ -33,8 +33,12 @@ import org.jclouds.compute.config.CustomizationResponse;
 import org.jclouds.compute.domain.Hardware;
 import org.jclouds.compute.domain.Image;
 import org.jclouds.compute.domain.NodeMetadata;
+import org.jclouds.compute.domain.NodeMetadataBuilder;
+import org.jclouds.compute.domain.NodeState;
 import org.jclouds.compute.domain.Template;
 import org.jclouds.compute.domain.TemplateBuilder;
+import org.jclouds.compute.predicates.AtomicNodeRunning;
+import org.jclouds.compute.strategy.GetNodeMetadataStrategy;
 import org.jclouds.compute.util.ComputeUtils;
 import org.jclouds.domain.Credentials;
 import org.jclouds.domain.Location;
@@ -50,9 +54,9 @@ import org.jclouds.ec2.domain.RunningInstance;
 import org.jclouds.ec2.options.RunInstancesOptions;
 import org.jclouds.ec2.services.ElasticIPAddressClient;
 import org.jclouds.ec2.services.InstanceClient;
+import org.testng.Assert;
 import org.testng.annotations.Test;
 
-import com.google.common.base.Predicate;
 import com.google.common.cache.LoadingCache;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Iterables;
@@ -73,17 +77,19 @@ public class EC2CreateNodesInGroupThenAddToSetTest {
 
       String imageId = "ami1";
       String instanceCreatedId = "instance1";
+      NodeMetadata nodeMetadata = new NodeMetadataBuilder().id(region + "/" + instanceCreatedId)
+            .providerId(instanceCreatedId).state(NodeState.RUNNING).build();
       // setup mocks
       TemplateBuilder templateBuilder = createMock(TemplateBuilder.class);
-      EC2CreateNodesInGroupThenAddToSet strategy = setupStrategy(templateBuilder);
+      EC2CreateNodesInGroupThenAddToSet strategy = setupStrategy(templateBuilder, nodeMetadata);
       InputParams input = new InputParams(location);
       InstanceClient instanceClient = createMock(InstanceClient.class);
       ElasticIPAddressClient ipClient = createMock(ElasticIPAddressClient.class);
       RunInstancesOptions ec2Options = createMock(RunInstancesOptions.class);
       RunningInstance instance = createMock(RunningInstance.class);
-      Reservation<? extends RunningInstance> reservation = new Reservation<RunningInstance>(region, ImmutableSet
+      Reservation<? extends RunningInstance> reservation = new Reservation<RunningInstance>(region,
-               .<String> of(), ImmutableSet.<RunningInstance> of(instance), "ownerId", "requesterId", "reservationId");
+            ImmutableSet.<String> of(), ImmutableSet.<RunningInstance> of(instance), "ownerId", "requesterId",
-      NodeMetadata nodeMetadata = createMock(NodeMetadata.class);
+            "reservationId");
 
       // enable auto-allocation
       strategy.autoAllocateElasticIps = true;
@@ -93,8 +99,8 @@ public class EC2CreateNodesInGroupThenAddToSetTest {
       expect(templateBuilder.build()).andReturn(input.template);
       expect(strategy.client.getInstanceServices()).andReturn(instanceClient).atLeastOnce();
       expect(
-               strategy.createKeyPairAndSecurityGroupsAsNeededAndReturncustomize.execute(region, input.tag,
+            strategy.createKeyPairAndSecurityGroupsAsNeededAndReturncustomize
-                        input.template)).andReturn(ec2Options);
+                  .execute(region, input.tag, input.template)).andReturn(ec2Options);
       expect(strategy.client.getElasticIPAddressServices()).andReturn(ipClient).atLeastOnce();
 
       expect(input.template.getLocation()).andReturn(input.location).atLeastOnce();
@@ -104,12 +110,11 @@ public class EC2CreateNodesInGroupThenAddToSetTest {
       // differences when ip allocation
       expect(ipClient.allocateAddressInRegion(region)).andReturn("1.1.1.1");
       expect(strategy.runningInstanceToNodeMetadata.apply(instance)).andReturn(nodeMetadata).atLeastOnce();
-      expect(strategy.nodeRunning.apply(nodeMetadata)).andReturn(true);
       ipClient.associateAddressInRegion(region, "1.1.1.1", instanceCreatedId);
       strategy.elasticIpCache.put(new RegionAndName(region, instanceCreatedId), "1.1.1.1");
 
       expect(instanceClient.runInstancesInRegion(region, zone, imageId, 1, input.count, ec2Options)).andReturn(
-               Reservation.class.cast(reservation));
+            Reservation.class.cast(reservation));
       expect(instance.getId()).andReturn(instanceCreatedId).atLeastOnce();
       // simulate a lazy credentials fetch
       Credentials creds = new Credentials("foo", "bar");
@@ -121,9 +126,9 @@ public class EC2CreateNodesInGroupThenAddToSetTest {
       expect(input.template.getOptions()).andReturn(input.options).atLeastOnce();
 
       expect(
-               strategy.utils.customizeNodesAndAddToGoodMapOrPutExceptionIntoBadMap(eq(input.options),
+            strategy.utils.customizeNodesAndAddToGoodMapOrPutExceptionIntoBadMap(eq(input.options),
-                        containsNodeMetadata(nodeMetadata), eq(input.nodes), eq(input.badNodes),
+                  containsNodeMetadata(nodeMetadata), eq(input.nodes), eq(input.badNodes), eq(input.customization)))
-                        eq(input.customization))).andReturn(null);
+            .andReturn(null);
 
       // replay mocks
       replay(templateBuilder);
@@ -131,7 +136,6 @@ public class EC2CreateNodesInGroupThenAddToSetTest {
       replay(ipClient);
       replay(ec2Options);
       replay(instance);
-      replay(nodeMetadata);
       input.replayMe();
       replayStrategy(strategy);
 
@@ -144,7 +148,6 @@ public class EC2CreateNodesInGroupThenAddToSetTest {
       verify(ipClient);
       verify(ec2Options);
       verify(instance);
-      verify(nodeMetadata);
       input.verifyMe();
       verifyStrategy(strategy);
    }
@@ -184,29 +187,32 @@ public class EC2CreateNodesInGroupThenAddToSetTest {
    private void assertRegionAndZoneForLocation(Location location, String region, String zone) {
       String imageId = "ami1";
       String instanceCreatedId = "instance1";
+      NodeMetadata nodeMetadata = new NodeMetadataBuilder().id(region + "/" + instanceCreatedId)
+            .providerId(instanceCreatedId).state(NodeState.RUNNING).build();
+
       // setup mocks
       TemplateBuilder templateBuilder = createMock(TemplateBuilder.class);
-      EC2CreateNodesInGroupThenAddToSet strategy = setupStrategy(templateBuilder);
+      EC2CreateNodesInGroupThenAddToSet strategy = setupStrategy(templateBuilder, nodeMetadata);
       InputParams input = new InputParams(location);
       InstanceClient instanceClient = createMock(InstanceClient.class);
       RunInstancesOptions ec2Options = createMock(RunInstancesOptions.class);
       RunningInstance instance = createMock(RunningInstance.class);
-      Reservation<? extends RunningInstance> reservation = new Reservation<RunningInstance>(region, ImmutableSet
+      Reservation<? extends RunningInstance> reservation = new Reservation<RunningInstance>(region,
-               .<String> of(), ImmutableSet.<RunningInstance> of(instance), "ownerId", "requesterId", "reservationId");
+            ImmutableSet.<String> of(), ImmutableSet.<RunningInstance> of(instance), "ownerId", "requesterId",
-      NodeMetadata nodeMetadata = createMock(NodeMetadata.class);
+            "reservationId");
 
       // setup expectations
       expect(templateBuilder.fromTemplate(input.template)).andReturn(templateBuilder);
       expect(templateBuilder.build()).andReturn(input.template);
       expect(strategy.client.getInstanceServices()).andReturn(instanceClient).atLeastOnce();
       expect(
-               strategy.createKeyPairAndSecurityGroupsAsNeededAndReturncustomize.execute(region, input.tag,
+            strategy.createKeyPairAndSecurityGroupsAsNeededAndReturncustomize
-                        input.template)).andReturn(ec2Options);
+                  .execute(region, input.tag, input.template)).andReturn(ec2Options);
       expect(input.template.getLocation()).andReturn(input.location).atLeastOnce();
       expect(input.template.getImage()).andReturn(input.image).atLeastOnce();
       expect(input.image.getProviderId()).andReturn(imageId).atLeastOnce();
       expect(instanceClient.runInstancesInRegion(region, zone, imageId, 1, input.count, ec2Options)).andReturn(
-               Reservation.class.cast(reservation));
+            Reservation.class.cast(reservation));
       expect(instance.getId()).andReturn(instanceCreatedId).atLeastOnce();
       // simulate a lazy credentials fetch
       Credentials creds = new Credentials("foo", "bar");
@@ -219,16 +225,15 @@ public class EC2CreateNodesInGroupThenAddToSetTest {
 
       expect(strategy.runningInstanceToNodeMetadata.apply(instance)).andReturn(nodeMetadata);
       expect(
-               strategy.utils.customizeNodesAndAddToGoodMapOrPutExceptionIntoBadMap(eq(input.options),
+            strategy.utils.customizeNodesAndAddToGoodMapOrPutExceptionIntoBadMap(eq(input.options),
-                        containsNodeMetadata(nodeMetadata), eq(input.nodes), eq(input.badNodes),
+                  containsNodeMetadata(nodeMetadata), eq(input.nodes), eq(input.badNodes), eq(input.customization)))
-                        eq(input.customization))).andReturn(null);
+            .andReturn(null);
 
       // replay mocks
       replay(templateBuilder);
       replay(instanceClient);
       replay(ec2Options);
       replay(instance);
-      replay(nodeMetadata);
       input.replayMe();
       replayStrategy(strategy);
 
@@ -240,16 +245,16 @@ public class EC2CreateNodesInGroupThenAddToSetTest {
       verify(instanceClient);
       verify(ec2Options);
       verify(instance);
-      verify(nodeMetadata);
       input.verifyMe();
       verifyStrategy(strategy);
    }
 
-   private static final Location REGION_AP_SOUTHEAST_1 = new LocationBuilder().scope(LocationScope.REGION).id(
+   private static final Location REGION_AP_SOUTHEAST_1 = new LocationBuilder().scope(LocationScope.REGION)
-            "ap-southeast-1").description("ap-southeast-1").parent(
+         .id("ap-southeast-1").description("ap-southeast-1")
-            new LocationBuilder().scope(LocationScope.PROVIDER).id("aws-ec2").description("aws-ec2").build()).build();
+         .parent(new LocationBuilder().scope(LocationScope.PROVIDER).id("aws-ec2").description("aws-ec2").build())
-   private static final Location ZONE_AP_SOUTHEAST_1A = new LocationBuilder().scope(LocationScope.ZONE).id(
+         .build();
-            "ap-southeast-1a").description("ap-southeast-1a").parent(REGION_AP_SOUTHEAST_1).build();
+   private static final Location ZONE_AP_SOUTHEAST_1A = new LocationBuilder().scope(LocationScope.ZONE)
+         .id("ap-southeast-1a").description("ap-southeast-1a").parent(REGION_AP_SOUTHEAST_1).build();
 
    // /////////////////////////////////////////////////////////////////////
    @SuppressWarnings("unchecked")
@@ -293,7 +298,6 @@ public class EC2CreateNodesInGroupThenAddToSetTest {
    private void verifyStrategy(EC2CreateNodesInGroupThenAddToSet strategy) {
       verify(strategy.createKeyPairAndSecurityGroupsAsNeededAndReturncustomize);
       verify(strategy.client);
-      verify(strategy.nodeRunning);
       verify(strategy.elasticIpCache);
       verify(strategy.instancePresent);
       verify(strategy.runningInstanceToNodeMetadata);
@@ -303,26 +307,33 @@ public class EC2CreateNodesInGroupThenAddToSetTest {
    }
 
    @SuppressWarnings("unchecked")
-   private EC2CreateNodesInGroupThenAddToSet setupStrategy(TemplateBuilder template) {
+   private EC2CreateNodesInGroupThenAddToSet setupStrategy(TemplateBuilder template, final NodeMetadata node) {
       EC2Client client = createMock(EC2Client.class);
       CreateKeyPairAndSecurityGroupsAsNeededAndReturnRunOptions createKeyPairAndSecurityGroupsAsNeededAndReturncustomize = createMock(CreateKeyPairAndSecurityGroupsAsNeededAndReturnRunOptions.class);
       InstancePresent instancePresent = createMock(InstancePresent.class);
       RunningInstanceToNodeMetadata runningInstanceToNodeMetadata = createMock(RunningInstanceToNodeMetadata.class);
       LoadingCache<RunningInstance, Credentials> instanceToCredentials = createMock(LoadingCache.class);
       LoadingCache<RegionAndName, String> elasticIpCache = createMock(LoadingCache.class);
-      Predicate<NodeMetadata> nodeRunning = createMock(Predicate.class);
+      GetNodeMetadataStrategy nodeRunning = new GetNodeMetadataStrategy(){
+
+         @Override
+         public NodeMetadata getNode(String input) {
+            Assert.assertEquals(input, node.getId());
+            return node;
+         }
+         
+      };
       Map<String, Credentials> credentialStore = createMock(Map.class);
       ComputeUtils utils = createMock(ComputeUtils.class);
-      return new EC2CreateNodesInGroupThenAddToSet(client, elasticIpCache, nodeRunning, Providers
+      return new EC2CreateNodesInGroupThenAddToSet(client, elasticIpCache, new AtomicNodeRunning(nodeRunning),
-               .<TemplateBuilder> of(template), createKeyPairAndSecurityGroupsAsNeededAndReturncustomize,
+            Providers.<TemplateBuilder> of(template), createKeyPairAndSecurityGroupsAsNeededAndReturncustomize,
-               instancePresent, runningInstanceToNodeMetadata, instanceToCredentials, credentialStore, utils);
+            instancePresent, runningInstanceToNodeMetadata, instanceToCredentials, credentialStore, utils);
    }
 
    private void replayStrategy(EC2CreateNodesInGroupThenAddToSet strategy) {
       replay(strategy.createKeyPairAndSecurityGroupsAsNeededAndReturncustomize);
       replay(strategy.client);
       replay(strategy.elasticIpCache);
-      replay(strategy.nodeRunning);
       replay(strategy.instancePresent);
       replay(strategy.runningInstanceToNodeMetadata);
       replay(strategy.instanceToCredentials);

common/trmk/src/main/java/org/jclouds/trmk/vcloud_0_8/compute/TerremarkVCloudComputeService.java

@@ -21,6 +21,7 @@ package org.jclouds.trmk.vcloud_0_8.compute;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.ExecutorService;
+import java.util.concurrent.atomic.AtomicReference;
 
 import javax.inject.Inject;
 import javax.inject.Named;
@@ -71,9 +72,9 @@ public class TerremarkVCloudComputeService extends BaseComputeService {
          RebootNodeStrategy rebootNodeStrategy, DestroyNodeStrategy destroyNodeStrategy,
          ResumeNodeStrategy resumeNodeStrategy, SuspendNodeStrategy suspendNodeStrategy,
          Provider<TemplateBuilder> templateBuilderProvider, Provider<TemplateOptions> templateOptionsProvider,
-         @Named("NODE_RUNNING") Predicate<NodeMetadata> nodeRunning,
+         @Named("NODE_RUNNING") Predicate<AtomicReference<NodeMetadata>> nodeRunning,
-         @Named("NODE_TERMINATED") Predicate<NodeMetadata> nodeTerminated,
+         @Named("NODE_TERMINATED") Predicate<AtomicReference<NodeMetadata>> nodeTerminated,
-         @Named("NODE_SUSPENDED") Predicate<NodeMetadata> nodeSuspended,
+         @Named("NODE_SUSPENDED") Predicate<AtomicReference<NodeMetadata>> nodeSuspended,
          InitializeRunScriptOnNodeOrPlaceInBadMap.Factory initScriptRunnerFactory,
          RunScriptOnNode.Factory runScriptOnNodeFactory, InitAdminAccess initAdminAccess,
          PersistNodeCredentials persistNodeCredentials, Timeouts timeouts,

compute/src/main/java/org/jclouds/compute/config/BaseComputeServiceContextModule.java

@@ -103,7 +103,7 @@ public abstract class BaseComputeServiceContextModule extends AbstractModule {
 
       install(new FactoryModuleBuilder().implement(new TypeLiteral<Callable<Void>>() {
       }, CustomizeNodeAndAddToGoodMapOrPutExceptionIntoBadMap.class)
-            .implement(new TypeLiteral<Function<NodeMetadata, Void>>() {
+            .implement(new TypeLiteral<Function<AtomicReference<NodeMetadata>, Void>>() {
             }, CustomizeNodeAndAddToGoodMapOrPutExceptionIntoBadMap.class)
             .build(CustomizeNodeAndAddToGoodMapOrPutExceptionIntoBadMap.Factory.class));
 

compute/src/main/java/org/jclouds/compute/config/ComputeServiceTimeoutsModule.java

@@ -20,10 +20,13 @@ package org.jclouds.compute.config;
 
 import static com.google.common.base.Predicates.not;
 
+import java.util.concurrent.atomic.AtomicReference;
+
 import javax.inject.Named;
 import javax.inject.Singleton;
 
 import org.jclouds.compute.domain.NodeMetadata;
+import org.jclouds.compute.predicates.AtomicNodeRunning;
 import org.jclouds.compute.predicates.NodeRunning;
 import org.jclouds.compute.predicates.NodeSuspended;
 import org.jclouds.compute.predicates.NodeTerminated;
@@ -46,6 +49,32 @@ public class ComputeServiceTimeoutsModule extends AbstractModule {
    @Provides
    @Singleton
    @Named("NODE_RUNNING")
+   protected Predicate<AtomicReference<NodeMetadata>> nodeRunning(AtomicNodeRunning stateRunning, Timeouts timeouts) {
+      return timeouts.nodeRunning == 0 ? stateRunning : new RetryablePredicate<AtomicReference<NodeMetadata>>(stateRunning,
+            timeouts.nodeRunning);
+   }
+
+   @Provides
+   @Singleton
+   @Named("NODE_TERMINATED")
+   protected Predicate<AtomicReference<NodeMetadata>> serverTerminated(AtomicNodeRunning stateTerminated, Timeouts timeouts) {
+      return timeouts.nodeTerminated == 0 ? stateTerminated : new RetryablePredicate<AtomicReference<NodeMetadata>>(stateTerminated,
+            timeouts.nodeTerminated);
+   }
+   
+
+   @Provides
+   @Singleton
+   @Named("NODE_SUSPENDED")
+   protected Predicate<AtomicReference<NodeMetadata>> serverSuspended(AtomicNodeRunning stateSuspended, Timeouts timeouts) {
+      return timeouts.nodeSuspended == 0 ? stateSuspended : new RetryablePredicate<AtomicReference<NodeMetadata>>(stateSuspended,
+            timeouts.nodeSuspended);
+   }
+   
+   @Provides
+   @Singleton
+   @Named("NODE_RUNNING")
+   @Deprecated
    protected Predicate<NodeMetadata> nodeRunning(NodeRunning stateRunning, Timeouts timeouts) {
       return timeouts.nodeRunning == 0 ? stateRunning : new RetryablePredicate<NodeMetadata>(stateRunning,
             timeouts.nodeRunning);
@@ -54,6 +83,7 @@ public class ComputeServiceTimeoutsModule extends AbstractModule {
    @Provides
    @Singleton
    @Named("NODE_TERMINATED")
+   @Deprecated
    protected Predicate<NodeMetadata> serverTerminated(NodeTerminated stateTerminated, Timeouts timeouts) {
       return timeouts.nodeTerminated == 0 ? stateTerminated : new RetryablePredicate<NodeMetadata>(stateTerminated,
             timeouts.nodeTerminated);
@@ -63,6 +93,7 @@ public class ComputeServiceTimeoutsModule extends AbstractModule {
    @Provides
    @Singleton
    @Named("NODE_SUSPENDED")
+   @Deprecated
    protected Predicate<NodeMetadata> serverSuspended(NodeSuspended stateSuspended, Timeouts timeouts) {
       return timeouts.nodeSuspended == 0 ? stateSuspended : new RetryablePredicate<NodeMetadata>(stateSuspended,
             timeouts.nodeSuspended);

compute/src/main/java/org/jclouds/compute/internal/BaseComputeService.java

@@ -125,9 +125,9 @@ public class BaseComputeService implements ComputeService {
    private final SuspendNodeStrategy suspendNodeStrategy;
    private final Provider<TemplateBuilder> templateBuilderProvider;
    private final Provider<TemplateOptions> templateOptionsProvider;
-   private final Predicate<NodeMetadata> nodeRunning;
+   private final Predicate<AtomicReference<NodeMetadata>> nodeRunning;
-   private final Predicate<NodeMetadata> nodeTerminated;
+   private final Predicate<AtomicReference<NodeMetadata>> nodeTerminated;
-   private final Predicate<NodeMetadata> nodeSuspended;
+   private final Predicate<AtomicReference<NodeMetadata>> nodeSuspended;
    private final InitializeRunScriptOnNodeOrPlaceInBadMap.Factory initScriptRunnerFactory;
    private final Timeouts timeouts;
    private final InitAdminAccess initAdminAccess;
@@ -143,9 +143,9 @@ public class BaseComputeService implements ComputeService {
          RebootNodeStrategy rebootNodeStrategy, DestroyNodeStrategy destroyNodeStrategy,
          ResumeNodeStrategy resumeNodeStrategy, SuspendNodeStrategy suspendNodeStrategy,
          Provider<TemplateBuilder> templateBuilderProvider, Provider<TemplateOptions> templateOptionsProvider,
-         @Named("NODE_RUNNING") Predicate<NodeMetadata> nodeRunning,
+         @Named("NODE_RUNNING") Predicate<AtomicReference<NodeMetadata>> nodeRunning,
-         @Named("NODE_TERMINATED") Predicate<NodeMetadata> nodeTerminated,
+         @Named("NODE_TERMINATED") Predicate<AtomicReference<NodeMetadata>> nodeTerminated,
-         @Named("NODE_SUSPENDED") Predicate<NodeMetadata> nodeSuspended,
+         @Named("NODE_SUSPENDED") Predicate<AtomicReference<NodeMetadata>> nodeSuspended,
          InitializeRunScriptOnNodeOrPlaceInBadMap.Factory initScriptRunnerFactory, InitAdminAccess initAdminAccess,
          RunScriptOnNode.Factory runScriptOnNodeFactory, PersistNodeCredentials persistNodeCredentials,
          Timeouts timeouts, @Named(Constants.PROPERTY_USER_THREADS) ExecutorService executor) {
@@ -285,7 +285,7 @@ public class BaseComputeService implements ComputeService {
 
       }, timeouts.nodeRunning, 1000, TimeUnit.MILLISECONDS);
       
-      boolean successful = tester.apply(id) && (node.get() == null || nodeTerminated.apply(node.get()));
+      boolean successful = tester.apply(id) && (node.get() == null || nodeTerminated.apply(node));
       if (successful)
          credentialStore.remove("node#" + id);
       logger.debug("<< destroyed node(%s) success(%s)", id, successful);
@@ -383,7 +383,7 @@ public class BaseComputeService implements ComputeService {
    public void rebootNode(String id) {
       checkNotNull(id, "id");
       logger.debug(">> rebooting node(%s)", id);
-      NodeMetadata node = rebootNodeStrategy.rebootNode(id);
+      AtomicReference<NodeMetadata> node = new AtomicReference<NodeMetadata>(rebootNodeStrategy.rebootNode(id));
       boolean successful = nodeRunning.apply(node);
       logger.debug("<< rebooted node(%s) success(%s)", id, successful);
    }
@@ -414,7 +414,7 @@ public class BaseComputeService implements ComputeService {
    public void resumeNode(String id) {
       checkNotNull(id, "id");
       logger.debug(">> resuming node(%s)", id);
-      NodeMetadata node = resumeNodeStrategy.resumeNode(id);
+      AtomicReference<NodeMetadata> node = new AtomicReference<NodeMetadata>(resumeNodeStrategy.resumeNode(id));
       boolean successful = nodeRunning.apply(node);
       logger.debug("<< resumed node(%s) success(%s)", id, successful);
    }
@@ -445,7 +445,7 @@ public class BaseComputeService implements ComputeService {
    public void suspendNode(String id) {
       checkNotNull(id, "id");
       logger.debug(">> suspending node(%s)", id);
-      NodeMetadata node = suspendNodeStrategy.suspendNode(id);
+      AtomicReference<NodeMetadata> node = new AtomicReference<NodeMetadata>(suspendNodeStrategy.suspendNode(id));
       boolean successful = nodeSuspended.apply(node);
       logger.debug("<< suspended node(%s) success(%s)", id, successful);
    }

compute/src/main/java/org/jclouds/compute/predicates/AtomicNodeRunning.java

@@ -0,0 +1,42 @@
+/**
+ * Licensed to jclouds, Inc. (jclouds) under one or more
+ * contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  jclouds licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.jclouds.compute.predicates;
+
+import javax.inject.Singleton;
+
+import org.jclouds.compute.domain.NodeState;
+import org.jclouds.compute.strategy.GetNodeMetadataStrategy;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.inject.Inject;
+
+/**
+ * 
+ * Tests to see if a node is running.
+ * 
+ * @author Adrian Cole
+ */
+@Singleton
+public class AtomicNodeRunning extends RefreshAndDoubleCheckOnFailUnlessStateInvalid {
+
+   @Inject
+   public AtomicNodeRunning(GetNodeMetadataStrategy client) {
+      super(NodeState.RUNNING, ImmutableSet.of(NodeState.ERROR, NodeState.TERMINATED), client);
+   }
+}

compute/src/main/java/org/jclouds/compute/predicates/AtomicNodeSuspended.java

@@ -0,0 +1,41 @@
+/**
+ * Licensed to jclouds, Inc. (jclouds) under one or more
+ * contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  jclouds licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.jclouds.compute.predicates;
+
+import javax.inject.Singleton;
+
+import org.jclouds.compute.domain.NodeState;
+import org.jclouds.compute.strategy.GetNodeMetadataStrategy;
+
+import com.google.inject.Inject;
+
+/**
+ * 
+ * Tests to see if a node is suspended.
+ * 
+ * @author Adrian Cole
+ */
+@Singleton
+public class AtomicNodeSuspended extends RefreshAndDoubleCheckOnFailUnlessStateInvalid {
+
+   @Inject
+   public AtomicNodeSuspended(GetNodeMetadataStrategy client) {
+      super(NodeState.SUSPENDED, client);
+   }
+}

compute/src/main/java/org/jclouds/compute/predicates/NodePresentAndInIntendedState.java

@@ -39,7 +39,9 @@ import com.google.inject.Inject;
  * Tests to see if a node is active.
  * 
  * @author Adrian Cole
+ * @see RefreshAndDoubleCheckOnFailUnlessStateInvalid
  */
+@Deprecated
 @Singleton
 public class NodePresentAndInIntendedState implements Predicate<NodeMetadata> {
 

compute/src/main/java/org/jclouds/compute/predicates/NodeRunning.java

@@ -31,8 +31,10 @@ import com.google.inject.Inject;
  * Tests to see if a node is running.
  * 
  * @author Adrian Cole
+ * @see AtomicNodeRunning
  */
 @Singleton
+@Deprecated
 public class NodeRunning extends NodePresentAndInIntendedState {
 
    @Inject

compute/src/main/java/org/jclouds/compute/predicates/NodeSuspended.java

@@ -30,8 +30,10 @@ import com.google.inject.Inject;
  * Tests to see if a node is suspended.
  * 
  * @author Adrian Cole
+ * @see AtomicNodeSuspended
  */
 @Singleton
+@Deprecated
 public class NodeSuspended extends NodePresentAndInIntendedState {
 
    @Inject

compute/src/main/java/org/jclouds/compute/predicates/NodeTerminated.java

@@ -36,8 +36,10 @@ import com.google.inject.Inject;
  * Tests to see if a node is deleted
  * 
  * @author Adrian Cole
+ * @see TrueIfNullOrTerminatedRefreshAndDoubleCheckOnFalse
  */
 @Singleton
+@Deprecated
 public class NodeTerminated implements Predicate<NodeMetadata> {
 
    private final GetNodeMetadataStrategy client;

compute/src/main/java/org/jclouds/compute/predicates/RefreshAndDoubleCheckOnFailUnlessStateInvalid.java

@@ -0,0 +1,86 @@
+/**
+ * Licensed to jclouds, Inc. (jclouds) under one or more
+ * contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  jclouds licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.jclouds.compute.predicates;
+
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicReference;
+
+import javax.annotation.Resource;
+import javax.inject.Singleton;
+
+import org.jclouds.compute.domain.NodeMetadata;
+import org.jclouds.compute.domain.NodeState;
+import org.jclouds.compute.strategy.GetNodeMetadataStrategy;
+import org.jclouds.logging.Logger;
+
+import com.google.common.base.Predicate;
+import com.google.common.collect.ImmutableSet;
+import com.google.inject.Inject;
+
+/**
+ * 
+ * Tests to see if a node is active.
+ * 
+ * @author Adrian Cole
+ */
+@Singleton
+public class RefreshAndDoubleCheckOnFailUnlessStateInvalid implements Predicate<AtomicReference<NodeMetadata>> {
+
+   private final GetNodeMetadataStrategy client;
+   private final NodeState intended;
+   private final Set<NodeState> invalids;
+   @Resource
+   protected Logger logger = Logger.NULL;
+
+   @Inject
+   public RefreshAndDoubleCheckOnFailUnlessStateInvalid(NodeState intended, GetNodeMetadataStrategy client) {
+      this(intended, ImmutableSet.of(NodeState.ERROR), client);
+   }
+
+   public RefreshAndDoubleCheckOnFailUnlessStateInvalid(NodeState intended, Set<NodeState> invalids, GetNodeMetadataStrategy client) {
+      this.intended = intended;
+      this.client = client;
+      this.invalids = invalids;
+   }
+
+   public boolean apply(AtomicReference<NodeMetadata> atomicNode) {
+      NodeMetadata node = atomicNode.get();
+      if (checkState(node))
+         return true;
+      node = refresh(node);
+      atomicNode.set(node);
+      return checkState(node);
+   }
+
+   public boolean checkState(NodeMetadata node) {
+      if (node == null)
+         return false;
+      logger.trace("%s: looking for node state %s: currently: %s", node.getId(), intended, node.getState());
+      if (invalids.contains(node.getState()))
+         throw new IllegalStateException("node " + node.getId() + " in location " + node.getLocation()
+                  + " is in invalid state "+node.getState());
+      return node.getState() == intended;
+   }
+
+   private NodeMetadata refresh(NodeMetadata node) {
+      if (node == null || node.getId() == null)
+         return null;
+      return client.getNode(node.getId());
+   }
+}

compute/src/main/java/org/jclouds/compute/predicates/TrueIfNullOrTerminatedRefreshAndDoubleCheckOnFalse.java

@@ -0,0 +1,69 @@
+/**
+ * Licensed to jclouds, Inc. (jclouds) under one or more
+ * contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  jclouds licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.jclouds.compute.predicates;
+
+import java.util.concurrent.atomic.AtomicReference;
+
+import javax.annotation.Resource;
+
+import org.jclouds.compute.domain.NodeMetadata;
+import org.jclouds.compute.domain.NodeState;
+import org.jclouds.compute.strategy.GetNodeMetadataStrategy;
+import org.jclouds.logging.Logger;
+
+import com.google.common.base.Predicate;
+import com.google.inject.Inject;
+
+/**
+ * 
+ * 
+ * @author Adrian Cole
+ */
+public class TrueIfNullOrTerminatedRefreshAndDoubleCheckOnFalse implements Predicate<AtomicReference<NodeMetadata>> {
+
+   private final GetNodeMetadataStrategy client;
+
+   @Resource
+   protected Logger logger = Logger.NULL;
+
+   @Inject
+   public TrueIfNullOrTerminatedRefreshAndDoubleCheckOnFalse(GetNodeMetadataStrategy client) {
+      this.client = client;
+   }
+
+   public boolean apply(AtomicReference<NodeMetadata> atomicNode) {
+      NodeMetadata node = atomicNode.get();
+      if (checkState(node))
+         return true;
+      node = refresh(node);
+      atomicNode.set(node);
+      return checkState(node);
+   }
+
+   public boolean checkState(NodeMetadata node) {
+      if (node == null)
+         return true;
+      logger.trace("%s: looking for node state %s: currently: %s", node.getId(), NodeState.TERMINATED, node.getState());
+      return node.getState() == NodeState.TERMINATED;
+   }
+
+   private NodeMetadata refresh(NodeMetadata node) {
+      return client.getNode(node.getId());
+   }
+}

compute/src/main/java/org/jclouds/compute/strategy/CustomizeNodeAndAddToGoodMapOrPutExceptionIntoBadMap.java

@@ -27,8 +27,8 @@ import static org.jclouds.compute.util.ComputeServiceUtils.findReachableSocketOn
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.Callable;
+import java.util.concurrent.atomic.AtomicReference;
 
-import org.jclouds.javax.annotation.Nullable;
 import javax.annotation.Resource;
 import javax.inject.Named;
 
@@ -41,6 +41,7 @@ import org.jclouds.compute.options.TemplateOptions;
 import org.jclouds.compute.predicates.RetryIfSocketNotYetOpen;
 import org.jclouds.compute.reference.ComputeServiceConstants;
 import org.jclouds.compute.reference.ComputeServiceConstants.Timeouts;
+import org.jclouds.javax.annotation.Nullable;
 import org.jclouds.logging.Logger;
 import org.jclouds.scriptbuilder.domain.Statement;
 
@@ -55,31 +56,28 @@ import com.google.inject.assistedinject.AssistedInject;
  * @author Adrian Cole
  */
 public class CustomizeNodeAndAddToGoodMapOrPutExceptionIntoBadMap implements Callable<Void>,
-         Function<NodeMetadata, Void> {
+      Function<AtomicReference<NodeMetadata>, Void> {
    public static interface Factory {
-      Callable<Void> create(TemplateOptions options, NodeMetadata node, Set<NodeMetadata> goodNodes,
+      Callable<Void> create(TemplateOptions options, AtomicReference<NodeMetadata> node, Set<NodeMetadata> goodNodes,
-               Map<NodeMetadata, Exception> badNodes,
+            Map<NodeMetadata, Exception> badNodes, Multimap<NodeMetadata, CustomizationResponse> customizationResponses);
-               Multimap<NodeMetadata, CustomizationResponse> customizationResponses);
 
-      Function<NodeMetadata, Void> create(TemplateOptions options, Set<NodeMetadata> goodNodes,
+      Function<AtomicReference<NodeMetadata>, Void> create(TemplateOptions options, Set<NodeMetadata> goodNodes,
-               Map<NodeMetadata, Exception> badNodes,
+            Map<NodeMetadata, Exception> badNodes, Multimap<NodeMetadata, CustomizationResponse> customizationResponses);
-               Multimap<NodeMetadata, CustomizationResponse> customizationResponses);
    }
 
    @Resource
    @Named(ComputeServiceConstants.COMPUTE_LOGGER)
    protected Logger logger = Logger.NULL;
 
-   private final Predicate<NodeMetadata> nodeRunning;
+   private final Predicate<AtomicReference<NodeMetadata>> nodeRunning;
    private final InitializeRunScriptOnNodeOrPlaceInBadMap.Factory initScriptRunnerFactory;
-   private final GetNodeMetadataStrategy getNode;
    private final RetryIfSocketNotYetOpen socketTester;
    private final Timeouts timeouts;
 
    @Nullable
    private final Statement statement;
    private final TemplateOptions options;
-   private NodeMetadata node;
+   private AtomicReference<NodeMetadata> node;
    private final Set<NodeMetadata> goodNodes;
    private final Map<NodeMetadata, Exception> badNodes;
    private final Multimap<NodeMetadata, CustomizationResponse> customizationResponses;
@@ -88,18 +86,17 @@ public class CustomizeNodeAndAddToGoodMapOrPutExceptionIntoBadMap implements Cal
 
    @AssistedInject
    public CustomizeNodeAndAddToGoodMapOrPutExceptionIntoBadMap(
-            @Named("NODE_RUNNING") Predicate<NodeMetadata> nodeRunning, GetNodeMetadataStrategy getNode,
+         @Named("NODE_RUNNING") Predicate<AtomicReference<NodeMetadata>> nodeRunning,
-            RetryIfSocketNotYetOpen socketTester, Timeouts timeouts,
+         RetryIfSocketNotYetOpen socketTester, Timeouts timeouts,
-            Function<TemplateOptions, Statement> templateOptionsToStatement,
+         Function<TemplateOptions, Statement> templateOptionsToStatement,
-            InitializeRunScriptOnNodeOrPlaceInBadMap.Factory initScriptRunnerFactory,
+         InitializeRunScriptOnNodeOrPlaceInBadMap.Factory initScriptRunnerFactory, @Assisted TemplateOptions options,
-            @Assisted TemplateOptions options, @Assisted @Nullable NodeMetadata node,
+         @Assisted AtomicReference<NodeMetadata> node, @Assisted Set<NodeMetadata> goodNodes,
-            @Assisted Set<NodeMetadata> goodNodes, @Assisted Map<NodeMetadata, Exception> badNodes,
+         @Assisted Map<NodeMetadata, Exception> badNodes,
-            @Assisted Multimap<NodeMetadata, CustomizationResponse> customizationResponses) {
+         @Assisted Multimap<NodeMetadata, CustomizationResponse> customizationResponses) {
       this.statement = checkNotNull(templateOptionsToStatement, "templateOptionsToStatement").apply(
-               checkNotNull(options, "options"));
+            checkNotNull(options, "options"));
       this.nodeRunning = checkNotNull(nodeRunning, "nodeRunning");
       this.initScriptRunnerFactory = checkNotNull(initScriptRunnerFactory, "initScriptRunnerFactory");
-      this.getNode = checkNotNull(getNode, "getNode");
       this.socketTester = checkNotNull(socketTester, "socketTester");
       this.timeouts = checkNotNull(timeouts, "timeouts");
       this.node = node;
@@ -111,61 +108,65 @@ public class CustomizeNodeAndAddToGoodMapOrPutExceptionIntoBadMap implements Cal
 
    @AssistedInject
    public CustomizeNodeAndAddToGoodMapOrPutExceptionIntoBadMap(
-            @Named("NODE_RUNNING") Predicate<NodeMetadata> nodeRunning, GetNodeMetadataStrategy getNode,
+         @Named("NODE_RUNNING") Predicate<AtomicReference<NodeMetadata>> nodeRunning, GetNodeMetadataStrategy getNode,
-            RetryIfSocketNotYetOpen socketTester, Timeouts timeouts,
+         RetryIfSocketNotYetOpen socketTester, Timeouts timeouts,
-            Function<TemplateOptions, Statement> templateOptionsToStatement,
+         Function<TemplateOptions, Statement> templateOptionsToStatement,
-            InitializeRunScriptOnNodeOrPlaceInBadMap.Factory initScriptRunnerFactory,
+         InitializeRunScriptOnNodeOrPlaceInBadMap.Factory initScriptRunnerFactory, @Assisted TemplateOptions options,
-            @Assisted TemplateOptions options, @Assisted Set<NodeMetadata> goodNodes,
+         @Assisted Set<NodeMetadata> goodNodes, @Assisted Map<NodeMetadata, Exception> badNodes,
-            @Assisted Map<NodeMetadata, Exception> badNodes,
+         @Assisted Multimap<NodeMetadata, CustomizationResponse> customizationResponses) {
-            @Assisted Multimap<NodeMetadata, CustomizationResponse> customizationResponses) {
+      this(nodeRunning, socketTester, timeouts, templateOptionsToStatement, initScriptRunnerFactory, options,
-      this(nodeRunning, getNode, socketTester, timeouts, templateOptionsToStatement, initScriptRunnerFactory, options,
+            new AtomicReference<NodeMetadata>(null), goodNodes, badNodes, customizationResponses);
-               null, goodNodes, badNodes, customizationResponses);
    }
 
    @Override
    public Void call() {
       checkState(!tainted, "this object is not designed to be reused: %s", toString());
       tainted = true;
-      String originalId = node.getId();
+      String originalId = node.get().getId();
-      NodeMetadata originalNode = node;
+      NodeMetadata originalNode = node.get();
       try {
          if (options.shouldBlockUntilRunning()) {
-            if (nodeRunning.apply(node)) {
+            try {
-               node = getNode.getNode(originalId);
+               if (!nodeRunning.apply(node)) {
-            } else {
+                  if (node.get() == null) {
-               NodeMetadata nodeForState = getNode.getNode(originalId);
+                     node.set(originalNode);
-               NodeState state = nodeForState == null ? NodeState.TERMINATED : nodeForState.getState();
+                     throw new IllegalStateException(format("api response for node(%s) was null, so we can't customize",
-               if (state == NodeState.TERMINATED)
+                           originalId));
+                  }
+                  throw new IllegalStateException(
+                        format(
+                              "node(%s) didn't achieve the state running within %d seconds, so we couldn't customize; final state: %s",
+                              originalId, timeouts.nodeRunning / 1000, node.get().getState()));
+               }
+            } catch (IllegalStateException e) {
+               if (node.get().getState() == NodeState.TERMINATED) {
                   throw new IllegalStateException(format("node(%s) terminated before we could customize", originalId));
-               else
+               } else {
-                  throw new IllegalStateException(format(
+                  throw e;
-                           "node(%s) didn't achieve the state running within %d seconds, final state: %s", originalId,
+               }
-                           timeouts.nodeRunning / 1000, state));
             }
-            if (node == null)
-               throw new IllegalStateException(format("node %s terminated before applying options", originalId));
             if (statement != null) {
-               RunScriptOnNode runner = initScriptRunnerFactory.create(node, statement, options, badNodes).call();
+               RunScriptOnNode runner = initScriptRunnerFactory.create(node.get(), statement, options, badNodes).call();
                if (runner != null) {
                   ExecResponse exec = runner.call();
-                  customizationResponses.put(node, exec);
+                  customizationResponses.put(node.get(), exec);
                }
             }
             if (options.getPort() > 0) {
-               findReachableSocketOnNode(socketTester.seconds(options.getSeconds()), node, options.getPort());
+               findReachableSocketOnNode(socketTester.seconds(options.getSeconds()), node.get(), options.getPort());
             }
          }
-         logger.debug("<< options applied node(%s)", originalId);
+         logger.debug("<< customized node(%s)", originalId);
-         goodNodes.add(node);
+         goodNodes.add(node.get());
       } catch (Exception e) {
-         logger.error(e, "<< problem applying options to node(%s): ", originalId, getRootCause(e).getMessage());
+         logger.error(e, "<< problem customizing node(%s): ", originalId, getRootCause(e).getMessage());
-         badNodes.put(node == null ? originalNode : node, e);
+         badNodes.put(node.get(), e);
       }
       return null;
    }
 
    @Override
-   public Void apply(NodeMetadata input) {
+   public Void apply(AtomicReference<NodeMetadata> input) {
       this.node = input;
       call();
       return null;

compute/src/main/java/org/jclouds/compute/strategy/impl/CreateNodesWithGroupEncodedIntoNameThenAddToSet.java

@@ -31,6 +31,7 @@ import java.util.Set;
 import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Future;
+import java.util.concurrent.atomic.AtomicReference;
 
 import javax.annotation.Resource;
 import javax.inject.Inject;
@@ -60,7 +61,7 @@ import com.google.common.collect.Multimap;
 @Singleton
 public class CreateNodesWithGroupEncodedIntoNameThenAddToSet implements CreateNodesInGroupThenAddToSet {
 
-   private class AddNode implements Callable<NodeMetadata> {
+   private class AddNode implements Callable<AtomicReference<NodeMetadata>> {
       private final String name;
       private final String group;
       private final Template template;
@@ -72,14 +73,14 @@ public class CreateNodesWithGroupEncodedIntoNameThenAddToSet implements CreateNo
       }
 
       @Override
-      public NodeMetadata call() throws Exception {
+      public AtomicReference<NodeMetadata> call() throws Exception {
          NodeMetadata node = null;
          logger.debug(">> adding node location(%s) name(%s) image(%s) hardware(%s)",
                   template.getLocation().getId(), name, template.getImage().getProviderId(), template.getHardware()
                            .getProviderId());
          node = addNodeWithGroupStrategy.createNodeWithGroupEncodedIntoName(group, name, template);
          logger.debug("<< %s node(%s)", node.getState(), node.getId());
-         return node;
+         return new AtomicReference<NodeMetadata>(node);
       }
 
       public String toString() {

compute/src/main/java/org/jclouds/compute/util/ComputeUtils.java

@@ -24,6 +24,7 @@ import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Future;
+import java.util.concurrent.atomic.AtomicReference;
 
 import javax.inject.Inject;
 import javax.inject.Named;
@@ -60,7 +61,7 @@ public class ComputeUtils {
       Map<NodeMetadata, Future<Void>> responses = newLinkedHashMap();
       for (NodeMetadata node : runningNodes) {
          responses.put(node, executor.submit(customizeNodeAndAddToGoodMapOrPutExceptionIntoBadMapFactory.create(
-                  options, node, goodNodes, badNodes, customizationResponses)));
+                  options, new AtomicReference<NodeMetadata>(node), goodNodes, badNodes, customizationResponses)));
       }
       return responses;
    }

compute/src/test/java/org/jclouds/compute/predicates/AtomicNodePredicatesTest.java

@@ -0,0 +1,144 @@
+/**
+ * Licensed to jclouds, Inc. (jclouds) under one or more
+ * contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  jclouds licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.jclouds.compute.predicates;
+
+import static org.easymock.EasyMock.createMock;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.replay;
+import static org.easymock.EasyMock.verify;
+
+import java.util.concurrent.atomic.AtomicReference;
+
+import org.jclouds.compute.domain.NodeMetadata;
+import org.jclouds.compute.domain.NodeMetadataBuilder;
+import org.jclouds.compute.domain.NodeState;
+import org.jclouds.compute.strategy.GetNodeMetadataStrategy;
+import org.testng.Assert;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+/**
+ * Tests possible uses of NodePredicates
+ * 
+ * @author Aled Sage, Adrian Cole
+ */
+@Test(singleThreaded = true, testName = "AtomicNodePredicatesTest")
+public class AtomicNodePredicatesTest {
+
+   private NodeMetadata node;
+   private GetNodeMetadataStrategy computeService;
+
+   @Test
+   public void testNoUpdatesAtomicReferenceOnPass() {
+      NodeMetadata running = new NodeMetadataBuilder().id("myid").state(NodeState.RUNNING).build();
+      GetNodeMetadataStrategy computeService = createMock(GetNodeMetadataStrategy.class);
+
+      replay(computeService);
+
+      AtomicNodeRunning nodeRunning = new AtomicNodeRunning(computeService);
+      AtomicReference<NodeMetadata> reference = new AtomicReference<NodeMetadata>(running);
+      Assert.assertTrue(nodeRunning.apply(reference));
+      Assert.assertEquals(reference.get(), running);
+
+      verify(computeService);
+
+   }
+
+   @Test
+   public void testRefreshUpdatesAtomicReferenceOnRecheckPending() {
+      NodeMetadata pending = new NodeMetadataBuilder().id("myid").state(NodeState.PENDING).build();
+      GetNodeMetadataStrategy computeService = createMock(GetNodeMetadataStrategy.class);
+
+      expect(computeService.getNode("myid")).andReturn(pending);
+
+      replay(computeService);
+
+      AtomicNodeRunning nodeRunning = new AtomicNodeRunning(computeService);
+      AtomicReference<NodeMetadata> reference = new AtomicReference<NodeMetadata>(pending);
+      Assert.assertFalse(nodeRunning.apply(reference));
+      Assert.assertEquals(reference.get(), pending);
+
+      verify(computeService);
+
+   }
+   @Test
+   public void testRefreshUpdatesAtomicReferenceOnRecheckRunning() {
+      NodeMetadata running = new NodeMetadataBuilder().id("myid").state(NodeState.RUNNING).build();
+      NodeMetadata pending = new NodeMetadataBuilder().id("myid").state(NodeState.PENDING).build();
+      GetNodeMetadataStrategy computeService = createMock(GetNodeMetadataStrategy.class);
+
+      expect(computeService.getNode("myid")).andReturn(running);
+
+      replay(computeService);
+
+      AtomicNodeRunning nodeRunning = new AtomicNodeRunning(computeService);
+      AtomicReference<NodeMetadata> reference = new AtomicReference<NodeMetadata>(pending);
+      Assert.assertTrue(nodeRunning.apply(reference));
+      Assert.assertEquals(reference.get(), running);
+
+      verify(computeService);
+
+   }
+
+   @BeforeMethod
+   public void setUp() throws Exception {
+      node = createMock(NodeMetadata.class);
+      computeService = createMock(GetNodeMetadataStrategy.class);
+
+      expect(node.getId()).andReturn("myid").anyTimes();
+      expect(computeService.getNode("myid")).andReturn(node).anyTimes();
+      expect(node.getLocation()).andReturn(null).anyTimes();
+   }
+
+   @Test
+   public void testNodeRunningReturnsTrueWhenRunning() {
+      expect(node.getState()).andReturn(NodeState.RUNNING).atLeastOnce();
+      replay(node);
+      replay(computeService);
+
+      AtomicNodeRunning nodeRunning = new AtomicNodeRunning(computeService);
+      AtomicReference<NodeMetadata> reference = new AtomicReference<NodeMetadata>(node);
+      Assert.assertTrue(nodeRunning.apply(reference));
+      Assert.assertEquals(reference.get(), node);
+   }
+
+   @Test(expectedExceptions = IllegalStateException.class)
+   public void testNodeRunningFailsOnTerminated() {
+      expect(node.getState()).andReturn(NodeState.TERMINATED).atLeastOnce();
+      replay(node);
+      replay(computeService);
+
+      AtomicNodeRunning nodeRunning = new AtomicNodeRunning(computeService);
+      AtomicReference<NodeMetadata> reference = new AtomicReference<NodeMetadata>(node);
+      nodeRunning.apply(reference);
+      Assert.assertEquals(reference.get(), node);
+   }
+
+   @Test(expectedExceptions = IllegalStateException.class)
+   public void testNodeRunningFailsOnError() {
+      expect(node.getState()).andReturn(NodeState.ERROR).atLeastOnce();
+      replay(node);
+      replay(computeService);
+
+      AtomicNodeRunning nodeRunning = new AtomicNodeRunning(computeService);
+      AtomicReference<NodeMetadata> reference = new AtomicReference<NodeMetadata>(node);
+      nodeRunning.apply(reference);
+      Assert.assertEquals(reference.get(), node);
+   }
+}

compute/src/test/java/org/jclouds/compute/strategy/CustomizeNodeAndAddToGoodMapOrPutExceptionIntoBadMapTest.java

@@ -18,14 +18,14 @@
  */
 package org.jclouds.compute.strategy;
 
-import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.createMock;
-import static org.easymock.classextension.EasyMock.createMock;
+import static org.easymock.EasyMock.replay;
-import static org.easymock.classextension.EasyMock.replay;
+import static org.easymock.EasyMock.verify;
-import static org.easymock.classextension.EasyMock.verify;
 import static org.testng.Assert.assertEquals;
 
 import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.atomic.AtomicReference;
 
 import org.jclouds.compute.config.CustomizationResponse;
 import org.jclouds.compute.domain.NodeMetadata;
@@ -33,13 +33,14 @@ import org.jclouds.compute.domain.NodeMetadataBuilder;
 import org.jclouds.compute.domain.NodeState;
 import org.jclouds.compute.functions.TemplateOptionsToStatement;
 import org.jclouds.compute.options.TemplateOptions;
+import org.jclouds.compute.predicates.AtomicNodeRunning;
 import org.jclouds.compute.predicates.RetryIfSocketNotYetOpen;
 import org.jclouds.compute.reference.ComputeServiceConstants.Timeouts;
 import org.jclouds.scriptbuilder.domain.Statement;
+import org.testng.Assert;
 import org.testng.annotations.Test;
 
 import com.google.common.base.Function;
-import com.google.common.base.Predicate;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.LinkedHashMultimap;
 import com.google.common.collect.Maps;
@@ -52,11 +53,8 @@ import com.google.common.collect.Sets;
 @Test(groups = "unit")
 public class CustomizeNodeAndAddToGoodMapOrPutExceptionIntoBadMapTest {
 
-   @SuppressWarnings("unchecked")
    public void testBreakWhenNodeStillPending() {
-      Predicate<NodeMetadata> nodeRunning = createMock(Predicate.class);
       InitializeRunScriptOnNodeOrPlaceInBadMap.Factory initScriptRunnerFactory = createMock(InitializeRunScriptOnNodeOrPlaceInBadMap.Factory.class);
-      GetNodeMetadataStrategy getNode = createMock(GetNodeMetadataStrategy.class);
       RetryIfSocketNotYetOpen socketTester = createMock(RetryIfSocketNotYetOpen.class);
       Timeouts timeouts = new Timeouts();
       Function<TemplateOptions, Statement> templateOptionsToStatement = new TemplateOptionsToStatement();
@@ -67,41 +65,39 @@ public class CustomizeNodeAndAddToGoodMapOrPutExceptionIntoBadMapTest {
       Map<NodeMetadata, Exception> badNodes = Maps.newLinkedHashMap();
       Multimap<NodeMetadata, CustomizationResponse> customizationResponses = LinkedHashMultimap.create();
 
-      NodeMetadata node = new NodeMetadataBuilder().ids("id").state(NodeState.PENDING).build();
+      final NodeMetadata node = new NodeMetadataBuilder().ids("id").state(NodeState.PENDING).build();
 
-      // node never reached running state
+      // node always stays pending
-      expect(nodeRunning.apply(node)).andReturn(false);
+      GetNodeMetadataStrategy nodeRunning = new GetNodeMetadataStrategy(){
-      expect(getNode.getNode(node.getId())).andReturn(node);
+
+         @Override
+         public NodeMetadata getNode(String input) {
+            Assert.assertEquals(input, node.getId());
+            return node;
+         }
+         
+      };
 
       // replay mocks
-      replay(nodeRunning);
+      replay(initScriptRunnerFactory, socketTester);
-      replay(initScriptRunnerFactory);
-      replay(getNode);
-      replay(socketTester);
-
       // run
-      new CustomizeNodeAndAddToGoodMapOrPutExceptionIntoBadMap(nodeRunning, getNode, socketTester, timeouts,
+      AtomicReference<NodeMetadata> atomicNode = new AtomicReference<NodeMetadata>(node);
-               templateOptionsToStatement, initScriptRunnerFactory, options, node, goodNodes, badNodes,
+      new CustomizeNodeAndAddToGoodMapOrPutExceptionIntoBadMap( new AtomicNodeRunning(nodeRunning),  socketTester, timeouts,
-               customizationResponses).apply(node);
+               templateOptionsToStatement, initScriptRunnerFactory, options, atomicNode, goodNodes, badNodes,
-
+               customizationResponses).apply(atomicNode);
+      
       assertEquals(goodNodes.size(), 0);
       assertEquals(badNodes.keySet(), ImmutableSet.of(node));
       assertEquals(badNodes.get(node).getMessage(),
-               "node(id) didn't achieve the state running within 1200 seconds, final state: PENDING");
+               "node(id) didn't achieve the state running within 1200 seconds, so we couldn't customize; final state: PENDING");
       assertEquals(customizationResponses.size(), 0);
 
       // verify mocks
-      verify(nodeRunning);
+      verify(initScriptRunnerFactory, socketTester);
-      verify(initScriptRunnerFactory);
-      verify(getNode);
-      verify(socketTester);
    }
 
-   @SuppressWarnings("unchecked")
    public void testBreakGraceFullyWhenNodeDied() {
-      Predicate<NodeMetadata> nodeRunning = createMock(Predicate.class);
       InitializeRunScriptOnNodeOrPlaceInBadMap.Factory initScriptRunnerFactory = createMock(InitializeRunScriptOnNodeOrPlaceInBadMap.Factory.class);
-      GetNodeMetadataStrategy getNode = createMock(GetNodeMetadataStrategy.class);
       RetryIfSocketNotYetOpen socketTester = createMock(RetryIfSocketNotYetOpen.class);
       Timeouts timeouts = new Timeouts();
       Function<TemplateOptions, Statement> templateOptionsToStatement = new TemplateOptionsToStatement();
@@ -112,32 +108,35 @@ public class CustomizeNodeAndAddToGoodMapOrPutExceptionIntoBadMapTest {
       Map<NodeMetadata, Exception> badNodes = Maps.newLinkedHashMap();
       Multimap<NodeMetadata, CustomizationResponse> customizationResponses = LinkedHashMultimap.create();
 
-      NodeMetadata node = new NodeMetadataBuilder().ids("id").state(NodeState.PENDING).build();
+      final NodeMetadata node = new NodeMetadataBuilder().ids("id").state(NodeState.PENDING).build();
+      final NodeMetadata deadNnode = new NodeMetadataBuilder().ids("id").state(NodeState.TERMINATED).build();
 
-      // node never reached running state
+      // node dies
-      expect(nodeRunning.apply(node)).andReturn(false);
+      GetNodeMetadataStrategy nodeRunning = new GetNodeMetadataStrategy(){
-      expect(getNode.getNode(node.getId())).andReturn(null);
+
+         @Override
+         public NodeMetadata getNode(String input) {
+            Assert.assertEquals(input, node.getId());
+            return deadNnode;
+         }
+         
+      };
 
       // replay mocks
-      replay(nodeRunning);
+      replay(initScriptRunnerFactory, socketTester);
-      replay(initScriptRunnerFactory);
-      replay(getNode);
-      replay(socketTester);
-
       // run
-      new CustomizeNodeAndAddToGoodMapOrPutExceptionIntoBadMap(nodeRunning, getNode, socketTester, timeouts,
+      AtomicReference<NodeMetadata> atomicNode = new AtomicReference<NodeMetadata>(node);
-               templateOptionsToStatement, initScriptRunnerFactory, options, node, goodNodes, badNodes,
+      new CustomizeNodeAndAddToGoodMapOrPutExceptionIntoBadMap( new AtomicNodeRunning(nodeRunning),  socketTester, timeouts,
-               customizationResponses).apply(node);
+               templateOptionsToStatement, initScriptRunnerFactory, options, atomicNode, goodNodes, badNodes,
+               customizationResponses).apply(atomicNode);
 
       assertEquals(goodNodes.size(), 0);
       assertEquals(badNodes.keySet(), ImmutableSet.of(node));
+      badNodes.get(node).printStackTrace();
       assertEquals(badNodes.get(node).getMessage(), "node(id) terminated before we could customize");
       assertEquals(customizationResponses.size(), 0);
 
       // verify mocks
-      verify(nodeRunning);
+      verify(initScriptRunnerFactory, socketTester);
-      verify(initScriptRunnerFactory);
-      verify(getNode);
-      verify(socketTester);
    }
 }

providers/aws-ec2/src/main/java/org/jclouds/aws/ec2/compute/AWSEC2ComputeService.java

@@ -25,9 +25,9 @@ import static org.jclouds.aws.ec2.reference.AWSEC2Constants.PROPERTY_EC2_GENERAT
 
 import java.util.Map;
 import java.util.Set;
-import java.util.Map.Entry;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.ExecutorService;
+import java.util.concurrent.atomic.AtomicReference;
 
 import javax.inject.Inject;
 import javax.inject.Named;
@@ -100,9 +100,9 @@ public class AWSEC2ComputeService extends EC2ComputeService {
             DestroyNodeStrategy destroyNodeStrategy, ResumeNodeStrategy startNodeStrategy,
             SuspendNodeStrategy stopNodeStrategy, Provider<TemplateBuilder> templateBuilderProvider,
             Provider<TemplateOptions> templateOptionsProvider,
-            @Named("NODE_RUNNING") Predicate<NodeMetadata> nodeRunning,
+            @Named("NODE_RUNNING") Predicate<AtomicReference<NodeMetadata>> nodeRunning,
-            @Named("NODE_TERMINATED") Predicate<NodeMetadata> nodeTerminated,
+            @Named("NODE_TERMINATED") Predicate<AtomicReference<NodeMetadata>> nodeTerminated,
-            @Named("NODE_SUSPENDED") Predicate<NodeMetadata> nodeSuspended,
+            @Named("NODE_SUSPENDED") Predicate<AtomicReference<NodeMetadata>> nodeSuspended,
             InitializeRunScriptOnNodeOrPlaceInBadMap.Factory initScriptRunnerFactory,
             RunScriptOnNode.Factory runScriptOnNodeFactory, InitAdminAccess initAdminAccess,
             PersistNodeCredentials persistNodeCredentials, Timeouts timeouts,

providers/aws-ec2/src/main/java/org/jclouds/aws/ec2/compute/strategy/AWSEC2CreateNodesInGroupThenAddToSet.java

@@ -23,6 +23,7 @@ import static com.google.common.collect.Iterables.transform;
 import static org.jclouds.aws.ec2.reference.AWSEC2Constants.PROPERTY_EC2_GENERATE_INSTANCE_NAMES;
 
 import java.util.Map;
+import java.util.concurrent.atomic.AtomicReference;
 
 import javax.annotation.Resource;
 import javax.inject.Inject;
@@ -79,7 +80,7 @@ public class AWSEC2CreateNodesInGroupThenAddToSet extends EC2CreateNodesInGroupT
    protected AWSEC2CreateNodesInGroupThenAddToSet(
             AWSEC2Client client,
             @Named("ELASTICIP") LoadingCache<RegionAndName, String> elasticIpCache,
-            @Named("NODE_RUNNING") Predicate<NodeMetadata> nodeRunning,
+            @Named("NODE_RUNNING") Predicate<AtomicReference<NodeMetadata>> nodeRunning,
             AWSEC2AsyncClient aclient,
             @Named(PROPERTY_EC2_GENERATE_INSTANCE_NAMES) boolean generateInstanceNames,
             Provider<TemplateBuilder> templateBuilderProvider,

providers/gogrid/src/main/java/org/jclouds/gogrid/compute/GoGridComputeService.java

@@ -21,6 +21,7 @@ package org.jclouds.gogrid.compute;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.ExecutorService;
+import java.util.concurrent.atomic.AtomicReference;
 
 import javax.inject.Inject;
 import javax.inject.Named;
@@ -68,9 +69,9 @@ public class GoGridComputeService extends BaseComputeService {
          RebootNodeStrategy rebootNodeStrategy, DestroyNodeStrategy destroyNodeStrategy,
          ResumeNodeStrategy resumeNodeStrategy, SuspendNodeStrategy suspendNodeStrategy,
          Provider<TemplateBuilder> templateBuilderProvider, Provider<TemplateOptions> templateOptionsProvider,
-         @Named("NODE_RUNNING") Predicate<NodeMetadata> nodeRunning,
+         @Named("NODE_RUNNING") Predicate<AtomicReference<NodeMetadata>> nodeRunning,
-         @Named("NODE_TERMINATED") Predicate<NodeMetadata> nodeTerminated,
+         @Named("NODE_TERMINATED") Predicate<AtomicReference<NodeMetadata>> nodeTerminated,
-         @Named("NODE_SUSPENDED") Predicate<NodeMetadata> nodeSuspended,
+         @Named("NODE_SUSPENDED") Predicate<AtomicReference<NodeMetadata>> nodeSuspended,
          InitializeRunScriptOnNodeOrPlaceInBadMap.Factory initScriptRunnerFactory, InitAdminAccess initAdminAccess,
          RunScriptOnNode.Factory runScriptOnNodeFactory, PersistNodeCredentials persistNodeCredentials, Timeouts timeouts,
          @Named(Constants.PROPERTY_USER_THREADS) ExecutorService executor) {