JCLOUDS-624 - Fixed bug in ListNodes

To fix this bug, I used the approach debated in the above issue: create
new methods to do the operation using an ExecutorService provided by the
user.The the old methods are still working, but now the operations in
those methods are not concurrent anymore.
This commit is contained in:
Luciano P. Sabenca(luciano.sabenca@movile.com) 2014-07-14 13:58:38 -03:00 committed by Ignasi Barrera
parent 1908f02e04
commit fc09c4eb46
19 changed files with 577 additions and 253 deletions

View File

@ -16,10 +16,8 @@
*/
package org.jclouds.chef;
import java.io.IOException;
import java.io.InputStream;
import java.util.List;
import com.google.common.io.InputSupplier;
import com.google.inject.ImplementedBy;
import org.jclouds.chef.domain.BootstrapConfig;
import org.jclouds.chef.domain.Client;
import org.jclouds.chef.domain.CookbookVersion;
@ -30,8 +28,10 @@ import org.jclouds.domain.JsonBall;
import org.jclouds.rest.annotations.SinceApiVersion;
import org.jclouds.scriptbuilder.domain.Statement;
import com.google.common.io.InputSupplier;
import com.google.inject.ImplementedBy;
import java.io.IOException;
import java.io.InputStream;
import java.util.List;
import java.util.concurrent.ExecutorService;
/**
* Provides high level Chef operations.
@ -98,7 +98,7 @@ public interface ChefService {
/**
* Gets the bootstrap configuration for a given group.
* <p>
* <p/>
* The bootstrap configuration is a Json object containing the run list and
* the configured attributes.
*
@ -158,6 +158,13 @@ public interface ChefService {
*/
Iterable<? extends Node> listNodes();
/**
* Lists the details of all existing nodes, executing concurrently using the executorService.
*
* @return The details of all existing nodes.
*/
Iterable<? extends Node> listNodes(ExecutorService executorService);
/**
* Lists the details of all existing nodes in the given environment.
*
@ -167,6 +174,16 @@ public interface ChefService {
@SinceApiVersion("0.10.0")
Iterable<? extends Node> listNodesInEnvironment(String environmentName);
/**
* Lists the details of all existing nodes in the given environment, using the ExecutorService to paralleling the execution.
*
* @param executorService The thread pool used in this operation
* @param environmentName The name fo the environment.
* @return The details of all existing nodes in the given environment.
*/
@SinceApiVersion("0.10.0")
Iterable<? extends Node> listNodesInEnvironment(String environmentName, ExecutorService executorService);
/**
* Lists the details of all existing clients.
*
@ -174,6 +191,13 @@ public interface ChefService {
*/
Iterable<? extends Client> listClients();
/**
* Lists the details of all existing clients, but executing concurrently using the threads available in the ExecutorService.
*
* @return The details of all existing clients.
*/
Iterable<? extends Client> listClients(ExecutorService executorService);
/**
* Lists the details of all existing cookbooks.
*
@ -181,6 +205,13 @@ public interface ChefService {
*/
Iterable<? extends CookbookVersion> listCookbookVersions();
/**
* Lists the details of all existing cookbooks. This method is executed concurrently, using the threads available in the ExecutorService.
*
* @return The details of all existing cookbooks.
*/
Iterable<? extends CookbookVersion> listCookbookVersions(ExecutorService executorService);
/**
* Lists the details of all existing cookbooks in an environment.
*
@ -189,6 +220,15 @@ public interface ChefService {
*/
Iterable<? extends CookbookVersion> listCookbookVersionsInEnvironment(String environmentName);
/**
* Lists the details of all existing cookbooks in an environment.
* @param executorService The thread pool to do the concurrent execution.
* @param environmentName The environment name.
* @return The details of all existing cookbooks in an environment.
*/
Iterable<? extends CookbookVersion> listCookbookVersionsInEnvironment(String environmentName, ExecutorService executorService);
/**
* Lists the details of all existing cookbooks in an environment
* limiting number of versions.
@ -200,6 +240,18 @@ public interface ChefService {
*/
Iterable<? extends CookbookVersion> listCookbookVersionsInEnvironment(String environmentName, String numVersions);
/**
* Lists the details of all existing cookbooks in an environment
* limiting number of versions.
*
* @param executorService The executorService used to do this operation concurrently.
* @param environmentName The environment name.
* @param numVersions The number of cookbook versions to include.
* Use 'all' to return all cookbook versions.
* @return The details of all existing cookbooks in environment.
*/
Iterable<? extends CookbookVersion> listCookbookVersionsInEnvironment(String environmentName, String numVersions, ExecutorService executorService);
/**
* Lists the details of all existing environments.
*

View File

@ -24,6 +24,7 @@ import java.io.InputStream;
import java.security.PrivateKey;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutorService;
import javax.annotation.Resource;
import javax.inject.Inject;
@ -50,9 +51,9 @@ import org.jclouds.chef.strategy.DeleteAllNodesInList;
import org.jclouds.chef.strategy.ListClients;
import org.jclouds.chef.strategy.ListCookbookVersions;
import org.jclouds.chef.strategy.ListCookbookVersionsInEnvironment;
import org.jclouds.chef.strategy.ListNodesInEnvironment;
import org.jclouds.chef.strategy.ListEnvironments;
import org.jclouds.chef.strategy.ListNodes;
import org.jclouds.chef.strategy.ListNodesInEnvironment;
import org.jclouds.chef.strategy.UpdateAutomaticAttributesOnNode;
import org.jclouds.crypto.Crypto;
import org.jclouds.domain.JsonBall;
@ -127,7 +128,7 @@ public class BaseChefService implements ChefService {
this.runListForGroup = checkNotNull(runListForGroup, "runListForGroup");
this.listEnvironments = checkNotNull(listEnvironments, "listEnvironments");
this.listNodesInEnvironment = checkNotNull(listNodesInEnvironment, "listNodesInEnvironment");
this.listCookbookVersionsInEnvironment = checkNotNull(listCookbookVersionsInEnvironment, "listCookbookVersionsInEnvironment");
this.listCookbookVersionsInEnvironment = checkNotNull(listCookbookVersionsInEnvironment,"listCookbookVersionsInEnvironment");
this.json = checkNotNull(json, "json");
this.crypto = checkNotNull(crypto, "crypto");
}
@ -232,26 +233,54 @@ public class BaseChefService implements ChefService {
return listNodes.execute();
}
@Override
public Iterable<? extends Node> listNodes(ExecutorService executorService) {
return listNodes.execute(executorService);
}
@Override
public Iterable<? extends Client> listClients() {
return listClients.execute();
}
@Override
public Iterable<? extends Client> listClients(ExecutorService executorService) {
return listClients.execute(executorService);
}
@Override
public Iterable<? extends CookbookVersion> listCookbookVersions() {
return listCookbookVersions.execute();
}
@Override public Iterable<? extends CookbookVersion> listCookbookVersions(
ExecutorService executorService) {
return listCookbookVersions.execute(executorService);
}
@Override
public Iterable<? extends CookbookVersion> listCookbookVersionsInEnvironment(String environmentName) {
return listCookbookVersionsInEnvironment.execute(environmentName);
}
@Override
public Iterable<? extends CookbookVersion> listCookbookVersionsInEnvironment(String environmentName, String numVersions) {
public Iterable<? extends CookbookVersion> listCookbookVersionsInEnvironment(String environmentName,
ExecutorService executorService) {
return listCookbookVersionsInEnvironment.execute(executorService, environmentName);
}
@Override
public Iterable<? extends CookbookVersion> listCookbookVersionsInEnvironment(String environmentName,
String numVersions) {
return listCookbookVersionsInEnvironment.execute(environmentName, numVersions);
}
@Override
public Iterable<? extends CookbookVersion> listCookbookVersionsInEnvironment(String environmentName,
String numVersions, ExecutorService executorService) {
return listCookbookVersionsInEnvironment.execute(executorService, environmentName, numVersions);
}
@Override
public Iterable<? extends Environment> listEnvironments() {
return listEnvironments.execute();
@ -262,4 +291,9 @@ public class BaseChefService implements ChefService {
return listNodesInEnvironment.execute(environmentName);
}
@Override
public Iterable<? extends Node> listNodesInEnvironment(String environmentName, ExecutorService executorService) {
return listNodesInEnvironment.execute(executorService, environmentName);
}
}

View File

@ -16,16 +16,17 @@
*/
package org.jclouds.chef.strategy;
import com.google.inject.ImplementedBy;
import org.jclouds.chef.domain.Client;
import org.jclouds.chef.strategy.internal.ListClientsImpl;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.inject.ImplementedBy;
import java.util.concurrent.ExecutorService;
@ImplementedBy(ListClientsImpl.class)
public interface ListClients {
Iterable<? extends Client> execute();
Iterable<? extends Client> execute(ListeningExecutorService executor);
Iterable<? extends Client> execute(ExecutorService executor);
}

View File

@ -16,16 +16,17 @@
*/
package org.jclouds.chef.strategy;
import com.google.inject.ImplementedBy;
import org.jclouds.chef.domain.CookbookVersion;
import org.jclouds.chef.strategy.internal.ListCookbookVersionsImpl;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.inject.ImplementedBy;
import java.util.concurrent.ExecutorService;
@ImplementedBy(ListCookbookVersionsImpl.class)
public interface ListCookbookVersions {
Iterable<? extends CookbookVersion> execute();
Iterable<? extends CookbookVersion> execute(ListeningExecutorService executor);
Iterable<? extends CookbookVersion> execute(ExecutorService executor);
}

View File

@ -16,11 +16,11 @@
*/
package org.jclouds.chef.strategy;
import com.google.inject.ImplementedBy;
import org.jclouds.chef.domain.CookbookVersion;
import org.jclouds.chef.strategy.internal.ListCookbookVersionsInEnvironmentImpl;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.inject.ImplementedBy;
import java.util.concurrent.ExecutorService;
@ImplementedBy(ListCookbookVersionsInEnvironmentImpl.class)
public interface ListCookbookVersionsInEnvironment {
@ -29,7 +29,9 @@ public interface ListCookbookVersionsInEnvironment {
Iterable<? extends CookbookVersion> execute(String environmentName, String numVersions);
Iterable<? extends CookbookVersion> execute(ListeningExecutorService executor, String environmentName);
Iterable<? extends CookbookVersion> execute(ExecutorService executor, String environmentName);
Iterable<? extends CookbookVersion> execute(ExecutorService executor, String environmentName, String numVersions);
Iterable<? extends CookbookVersion> execute(ListeningExecutorService executor, String environmentName, String numVersions);
}

View File

@ -16,16 +16,16 @@
*/
package org.jclouds.chef.strategy;
import com.google.inject.ImplementedBy;
import org.jclouds.chef.domain.Environment;
import org.jclouds.chef.strategy.internal.ListEnvironmentsImpl;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.inject.ImplementedBy;
import java.util.concurrent.ExecutorService;
@ImplementedBy(ListEnvironmentsImpl.class)
public interface ListEnvironments {
Iterable<? extends Environment> execute();
Iterable<? extends Environment> execute(ListeningExecutorService executor);
Iterable<? extends Environment> execute(ExecutorService executor);
}

View File

@ -16,16 +16,17 @@
*/
package org.jclouds.chef.strategy;
import com.google.inject.ImplementedBy;
import org.jclouds.chef.domain.Node;
import org.jclouds.chef.strategy.internal.ListNodesImpl;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.inject.ImplementedBy;
import java.util.concurrent.ExecutorService;
@ImplementedBy(ListNodesImpl.class)
public interface ListNodes {
Iterable<? extends Node> execute();
Iterable<? extends Node> execute(ListeningExecutorService executor);
Iterable<? extends Node> execute(ExecutorService executor);
}

View File

@ -16,16 +16,17 @@
*/
package org.jclouds.chef.strategy;
import com.google.inject.ImplementedBy;
import org.jclouds.chef.domain.Node;
import org.jclouds.chef.strategy.internal.ListNodesInEnvironmentImpl;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.inject.ImplementedBy;
import java.util.concurrent.ExecutorService;
@ImplementedBy(ListNodesInEnvironmentImpl.class)
public interface ListNodesInEnvironment {
Iterable<? extends Node> execute(String environmentName);
Iterable<? extends Node> execute(ListeningExecutorService executor, String environmentName);
Iterable<? extends Node> execute(ExecutorService executor, String environmentName);
}

View File

@ -0,0 +1,97 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jclouds.chef.strategy.internal;
import com.google.common.base.Function;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.ListeningExecutorService;
import org.jclouds.chef.ChefApi;
import org.jclouds.chef.domain.CookbookVersion;
import org.jclouds.logging.Logger;
import java.util.List;
import java.util.Set;
import java.util.concurrent.Callable;
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.collect.Iterables.concat;
import static com.google.common.collect.Iterables.transform;
import static com.google.common.util.concurrent.Futures.allAsList;
import static com.google.common.util.concurrent.Futures.getUnchecked;
public abstract class BaseListCookbookVersionsImpl {
protected final ChefApi api;
protected Logger logger = Logger.NULL;
BaseListCookbookVersionsImpl(ChefApi api) {
this.api = checkNotNull(api, "api");
}
protected Iterable<? extends CookbookVersion> execute(Iterable<String> toGet) {
return concat(transform(toGet, new Function<String, Iterable<? extends CookbookVersion>>() {
@Override
public Iterable<? extends CookbookVersion> apply(final String cookbook) {
// TODO getting each version could also go parallel
Set<String> cookbookVersions = api.listVersionsOfCookbook(cookbook);
Iterable<? extends CookbookVersion> cookbooksVersions = transform(cookbookVersions,
new Function<String, CookbookVersion>() {
@Override
public CookbookVersion apply(final String version) {
return api.getCookbook(cookbook, version);
}
}
);
logger.trace(String.format("getting versions of cookbook: %s", cookbook));
return cookbooksVersions;
}
}));
}
protected Iterable<? extends CookbookVersion> executeConcurrently(final ListeningExecutorService executor,
Iterable<String> cookbookNames) {
return concat(transform(cookbookNames, new Function<String, Iterable<? extends CookbookVersion>>() {
@Override
public Iterable<? extends CookbookVersion> apply(final String cookbook) {
// TODO getting each version could also go parallel
Set<String> cookbookVersions = api.listVersionsOfCookbook(cookbook);
ListenableFuture<List<CookbookVersion>> futures = allAsList(transform(cookbookVersions,
new Function<String, ListenableFuture<CookbookVersion>>() {
@Override
public ListenableFuture<CookbookVersion> apply(final String version) {
return executor.submit(new Callable<CookbookVersion>() {
@Override
public CookbookVersion call() throws Exception {
return api.getCookbook(cookbook, version);
}
});
}
}
));
logger.trace(String.format("getting versions of cookbook: %s", cookbook));
return getUnchecked(futures);
}
}));
}
}

View File

@ -0,0 +1,77 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jclouds.chef.strategy.internal;
import com.google.common.base.Function;
import com.google.common.base.Joiner;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.ListeningExecutorService;
import org.jclouds.chef.ChefApi;
import org.jclouds.chef.domain.Node;
import org.jclouds.logging.Logger;
import java.util.List;
import java.util.concurrent.Callable;
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.collect.Iterables.transform;
import static com.google.common.util.concurrent.Futures.allAsList;
import static com.google.common.util.concurrent.Futures.getUnchecked;
public abstract class BaseListNodesImpl {
protected final ChefApi api;
protected Logger logger = Logger.NULL;
BaseListNodesImpl(ChefApi api) {
this.api = checkNotNull(api, "api");
}
protected Iterable<? extends Node> execute(Iterable<String> toGet) {
Iterable<? extends Node> nodes = transform(toGet, new Function<String, Node>() {
@Override
public Node apply(final String input) {
return api.getNode(input);
}
}
);
logger.trace(String.format("getting nodes: %s", Joiner.on(',').join(toGet)));
return nodes;
}
protected Iterable<? extends Node> executeConcurrently(final ListeningExecutorService executor,
Iterable<String> toGet) {
ListenableFuture<List<Node>> futures = allAsList(transform(toGet, new Function<String, ListenableFuture<Node>>() {
@Override
public ListenableFuture<Node> apply(final String input) {
return executor.submit(new Callable<Node>() {
@Override
public Node call() throws Exception {
return api.getNode(input);
}
});
}
}));
logger.trace(String.format("getting nodes: %s", Joiner.on(',').join(toGet)));
return getUnchecked(futures);
}
}

View File

@ -28,45 +28,66 @@ import javax.annotation.Resource;
import javax.inject.Named;
import javax.inject.Singleton;
import org.jclouds.Constants;
import com.google.common.base.Function;
import com.google.common.base.Joiner;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.common.util.concurrent.MoreExecutors;
import com.google.inject.Inject;
import org.jclouds.chef.ChefApi;
import org.jclouds.chef.config.ChefProperties;
import org.jclouds.chef.domain.Client;
import org.jclouds.chef.strategy.ListClients;
import org.jclouds.logging.Logger;
import com.google.common.base.Function;
import com.google.common.base.Joiner;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.inject.Inject;
import java.util.concurrent.ExecutorService;
@Singleton
public class ListClientsImpl implements ListClients {
protected final ChefApi api;
protected final ListeningExecutorService userExecutor;
@Resource
@Named(ChefProperties.CHEF_LOGGER)
protected Logger logger = Logger.NULL;
@Inject
ListClientsImpl(@Named(Constants.PROPERTY_USER_THREADS) ListeningExecutorService userExecutor, ChefApi api) {
this.userExecutor = checkNotNull(userExecutor, "userExecuor");
ListClientsImpl(ChefApi api) {
this.api = checkNotNull(api, "api");
}
@Override
public Iterable<? extends Client> execute() {
return execute(userExecutor);
Iterable<String> toGet = api.listClients();
Iterable<? extends Client> clients = transform(toGet,
new Function<String, Client>() {
@Override
public Client apply(final String input) {
return api.getClient(input);
}
}
);
logger.trace(String.format("getting clients: %s", Joiner.on(',').join(toGet)));
return clients;
}
@Override
public Iterable<? extends Client> execute(ListeningExecutorService executor) {
return execute(executor, api.listClients());
public Iterable<? extends Client> execute(ExecutorService executorService) {
return this.execute(MoreExecutors.listeningDecorator(executorService));
}
private Iterable<? extends Client> execute(final ListeningExecutorService executor, Iterable<String> toGet) {
private Iterable<? extends Client> execute(ListeningExecutorService listeningExecutor) {
return executeConcurrently(listeningExecutor, api.listClients());
}
private Iterable<? extends Client> executeConcurrently(final ListeningExecutorService executor,
Iterable<String> toGet) {
ListenableFuture<List<Client>> futures = allAsList(transform(toGet,
new Function<String, ListenableFuture<Client>>() {
@Override
@ -78,7 +99,8 @@ public class ListClientsImpl implements ListClients {
}
});
}
}));
}
));
logger.trace(String.format("getting clients: %s", Joiner.on(',').join(toGet)));
return getUnchecked(futures);

View File

@ -16,81 +16,45 @@
*/
package org.jclouds.chef.strategy.internal;
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.collect.Iterables.concat;
import static com.google.common.collect.Iterables.transform;
import static com.google.common.util.concurrent.Futures.allAsList;
import static com.google.common.util.concurrent.Futures.getUnchecked;
import java.util.List;
import java.util.Set;
import java.util.concurrent.Callable;
import javax.annotation.Resource;
import javax.inject.Named;
import javax.inject.Singleton;
import org.jclouds.Constants;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.common.util.concurrent.MoreExecutors;
import com.google.inject.Inject;
import org.jclouds.chef.ChefApi;
import org.jclouds.chef.config.ChefProperties;
import org.jclouds.chef.domain.CookbookVersion;
import org.jclouds.chef.strategy.ListCookbookVersions;
import org.jclouds.logging.Logger;
import com.google.common.base.Function;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.inject.Inject;
import javax.annotation.Resource;
import javax.inject.Named;
import javax.inject.Singleton;
import java.util.concurrent.ExecutorService;
@Singleton
public class ListCookbookVersionsImpl implements ListCookbookVersions {
public class ListCookbookVersionsImpl extends BaseListCookbookVersionsImpl implements ListCookbookVersions {
protected final ChefApi api;
protected final ListeningExecutorService userExecutor;
@Resource
@Named(ChefProperties.CHEF_LOGGER)
protected Logger logger = Logger.NULL;
@Inject
ListCookbookVersionsImpl(@Named(Constants.PROPERTY_USER_THREADS) ListeningExecutorService userExecutor, ChefApi api) {
this.userExecutor = checkNotNull(userExecutor, "userExecuor");
this.api = checkNotNull(api, "api");
ListCookbookVersionsImpl(ChefApi api) {
super(api);
}
@Override
public Iterable<? extends CookbookVersion> execute() {
return execute(userExecutor);
return super.execute(api.listCookbooks());
}
@Override
public Iterable<? extends CookbookVersion> execute(ListeningExecutorService executor) {
return execute(executor, api.listCookbooks());
public Iterable<? extends CookbookVersion> execute(ExecutorService executor) {
return this.executeConcurrently(MoreExecutors.listeningDecorator(executor));
}
private Iterable<? extends CookbookVersion> execute(final ListeningExecutorService executor,
Iterable<String> cookbookNames) {
return concat(transform(cookbookNames, new Function<String, Iterable<? extends CookbookVersion>>() {
@Override
public Iterable<? extends CookbookVersion> apply(final String cookbook) {
// TODO getting each version could also go parallel
Set<String> cookbookVersions = api.listVersionsOfCookbook(cookbook);
ListenableFuture<List<CookbookVersion>> futures = allAsList(transform(cookbookVersions,
new Function<String, ListenableFuture<CookbookVersion>>() {
@Override
public ListenableFuture<CookbookVersion> apply(final String version) {
return executor.submit(new Callable<CookbookVersion>() {
@Override
public CookbookVersion call() throws Exception {
return api.getCookbook(cookbook, version);
private Iterable<? extends CookbookVersion> executeConcurrently(ListeningExecutorService executor) {
return super.executeConcurrently(executor, api.listCookbooks());
}
});
}
}));
logger.trace(String.format("getting versions of cookbook: %s", cookbook));
return getUnchecked(futures);
}
}));
}
}

View File

@ -16,21 +16,16 @@
*/
package org.jclouds.chef.strategy.internal;
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.collect.Iterables.concat;
import static com.google.common.collect.Iterables.transform;
import static com.google.common.util.concurrent.Futures.allAsList;
import static com.google.common.util.concurrent.Futures.getUnchecked;
import java.util.List;
import java.util.Set;
import java.util.concurrent.Callable;
import javax.annotation.Resource;
import javax.inject.Named;
import javax.inject.Singleton;
import org.jclouds.Constants;
import com.google.common.base.Function;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.common.util.concurrent.MoreExecutors;
import com.google.inject.Inject;
import org.jclouds.chef.ChefApi;
import org.jclouds.chef.config.ChefProperties;
import org.jclouds.chef.domain.CookbookDefinition;
@ -38,69 +33,85 @@ import org.jclouds.chef.domain.CookbookVersion;
import org.jclouds.chef.strategy.ListCookbookVersionsInEnvironment;
import org.jclouds.logging.Logger;
import com.google.common.base.Function;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.inject.Inject;
import java.util.concurrent.ExecutorService;
@Singleton
public class ListCookbookVersionsInEnvironmentImpl implements ListCookbookVersionsInEnvironment {
public class ListCookbookVersionsInEnvironmentImpl extends BaseListCookbookVersionsImpl
implements ListCookbookVersionsInEnvironment {
protected final ChefApi api;
protected final ListeningExecutorService userExecutor;
@Resource
@Named(ChefProperties.CHEF_LOGGER)
protected Logger logger = Logger.NULL;
@Inject
ListCookbookVersionsInEnvironmentImpl(@Named(Constants.PROPERTY_USER_THREADS) ListeningExecutorService userExecutor, ChefApi api) {
this.userExecutor = checkNotNull(userExecutor, "userExecuor");
this.api = checkNotNull(api, "api");
ListCookbookVersionsInEnvironmentImpl(ChefApi api) {
super(api);
}
@Override
public Iterable<? extends CookbookVersion> execute(String environmentName) {
return execute(userExecutor, environmentName);
return super.execute(transform(api.listCookbooksInEnvironment(environmentName),
new Function<CookbookDefinition, String>() {
@Override
public String apply(CookbookDefinition cookbookDefinition) {
return cookbookDefinition.getName();
}
}
));
}
@Override
public Iterable<? extends CookbookVersion> execute(String environmentName, String numVersions) {
return execute(userExecutor, environmentName, numVersions);
}
return super.execute(transform(api.listCookbooksInEnvironment(environmentName, numVersions),
new Function<CookbookDefinition, String>() {
public Iterable<? extends CookbookVersion> execute(ListeningExecutorService executor, String environmentName) {
return execute(executor, api.listCookbooksInEnvironment(environmentName));
@Override
public String apply(CookbookDefinition cookbookDefinition) {
return cookbookDefinition.getName();
}
}
));
}
@Override
public Iterable<? extends CookbookVersion> execute(ListeningExecutorService executor, String environmentName, String numVersions) {
return execute(executor, api.listCookbooksInEnvironment(environmentName, numVersions));
public Iterable<? extends CookbookVersion> execute(ExecutorService executor,
String environmentName) {
return this.executeConcurrently(MoreExecutors.listeningDecorator(executor), environmentName);
}
private Iterable<? extends CookbookVersion> execute(final ListeningExecutorService executor,
Iterable<CookbookDefinition> cookbookDefs) {
return concat(transform(cookbookDefs, new Function<CookbookDefinition, Iterable<? extends CookbookVersion>>() {
@Override
public Iterable<? extends CookbookVersion> execute(ExecutorService executor,
String environmentName, String numVersions) {
return this.executeConcurrently(MoreExecutors.listeningDecorator(executor), environmentName, numVersions);
}
private Iterable<? extends CookbookVersion> executeConcurrently(ListeningExecutorService executor,
String environmentName) {
return super.execute(
transform(api.listCookbooksInEnvironment(environmentName), new Function<CookbookDefinition, String>() {
@Override
public Iterable<? extends CookbookVersion> apply(final CookbookDefinition cookbookDef) {
// TODO getting each version could also go parallel
Set<CookbookDefinition.Version> cookbookVersions = cookbookDef.getVersions();
ListenableFuture<List<CookbookVersion>> futures = allAsList(transform(cookbookVersions,
new Function<CookbookDefinition.Version, ListenableFuture<CookbookVersion>>() {
@Override
public ListenableFuture<CookbookVersion> apply(final CookbookDefinition.Version version) {
return executor.submit(new Callable<CookbookVersion>() {
@Override
public CookbookVersion call() throws Exception {
return api.getCookbook(cookbookDef.getName(), version.getVersion());
public String apply(CookbookDefinition cookbookDefinition) {
return cookbookDefinition.getName();
}
});
})
);
}
}));
logger.trace(String.format("getting versions of cookbook %s: ", cookbookDef.getName()));
return getUnchecked(futures);
private Iterable<? extends CookbookVersion> executeConcurrently(ListeningExecutorService executor,
String environmentName, String numVersions) {
return super.execute(transform(api.listCookbooksInEnvironment(environmentName, numVersions),
new Function<CookbookDefinition, String>() {
@Override
public String apply(CookbookDefinition cookbookDefinition) {
return cookbookDefinition.getName();
}
}));
}
));
}
}

View File

@ -40,6 +40,12 @@ import com.google.common.base.Joiner;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.inject.Inject;
import com.google.common.util.concurrent.MoreExecutors;
import java.util.concurrent.ExecutorService;
@Singleton
public class ListEnvironmentsImpl implements ListEnvironments {
@ -62,7 +68,11 @@ public class ListEnvironmentsImpl implements ListEnvironments {
}
@Override
public Iterable<? extends Environment> execute(ListeningExecutorService executor) {
public Iterable<? extends Environment> execute(ExecutorService executor) {
return this.execute(MoreExecutors.listeningDecorator(executor));
}
private Iterable<? extends Environment> execute(ListeningExecutorService executor) {
return execute(executor, api.listEnvironments());
}

View File

@ -16,71 +16,47 @@
*/
package org.jclouds.chef.strategy.internal;
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.collect.Iterables.transform;
import static com.google.common.util.concurrent.Futures.allAsList;
import static com.google.common.util.concurrent.Futures.getUnchecked;
import java.util.List;
import java.util.concurrent.Callable;
import javax.annotation.Resource;
import javax.inject.Named;
import javax.inject.Singleton;
import org.jclouds.Constants;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.common.util.concurrent.MoreExecutors;
import com.google.inject.Inject;
import org.jclouds.chef.ChefApi;
import org.jclouds.chef.config.ChefProperties;
import org.jclouds.chef.domain.Node;
import org.jclouds.chef.strategy.ListNodes;
import org.jclouds.logging.Logger;
import com.google.common.base.Function;
import com.google.common.base.Joiner;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.inject.Inject;
import java.util.concurrent.ExecutorService;
@Singleton
public class ListNodesImpl implements ListNodes {
public class ListNodesImpl extends BaseListNodesImpl implements ListNodes {
protected final ChefApi api;
protected final ListeningExecutorService userExecutor;
@Resource
@Named(ChefProperties.CHEF_LOGGER)
protected Logger logger = Logger.NULL;
@Inject
ListNodesImpl(@Named(Constants.PROPERTY_USER_THREADS) ListeningExecutorService userExecutor, ChefApi api) {
this.userExecutor = checkNotNull(userExecutor, "userExecuor");
this.api = checkNotNull(api, "api");
ListNodesImpl(ChefApi api) {
super(api);
}
@Override
public Iterable<? extends Node> execute() {
return execute(userExecutor);
return super.execute(api.listNodes());
}
@Override
public Iterable<? extends Node> execute(ListeningExecutorService executor) {
return execute(executor, api.listNodes());
public Iterable<? extends Node> execute(ExecutorService executor) {
return this.executeConcurrently(MoreExecutors.listeningDecorator(executor));
}
private Iterable<? extends Node> execute(final ListeningExecutorService executor, Iterable<String> toGet) {
ListenableFuture<List<Node>> futures = allAsList(transform(toGet, new Function<String, ListenableFuture<Node>>() {
@Override
public ListenableFuture<Node> apply(final String input) {
return executor.submit(new Callable<Node>() {
@Override
public Node call() throws Exception {
return api.getNode(input);
}
});
}
}));
logger.trace(String.format("getting nodes: %s", Joiner.on(',').join(toGet)));
return getUnchecked(futures);
private Iterable<? extends Node> executeConcurrently(ListeningExecutorService executor) {
return super.executeConcurrently(executor, api.listNodes());
}
}

View File

@ -16,71 +16,47 @@
*/
package org.jclouds.chef.strategy.internal;
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.collect.Iterables.transform;
import static com.google.common.util.concurrent.Futures.allAsList;
import static com.google.common.util.concurrent.Futures.getUnchecked;
import java.util.List;
import java.util.concurrent.Callable;
import javax.annotation.Resource;
import javax.inject.Named;
import javax.inject.Singleton;
import org.jclouds.Constants;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.common.util.concurrent.MoreExecutors;
import com.google.inject.Inject;
import org.jclouds.chef.ChefApi;
import org.jclouds.chef.config.ChefProperties;
import org.jclouds.chef.domain.Node;
import org.jclouds.chef.strategy.ListNodesInEnvironment;
import org.jclouds.logging.Logger;
import com.google.common.base.Function;
import com.google.common.base.Joiner;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.inject.Inject;
import java.util.concurrent.ExecutorService;
@Singleton
public class ListNodesInEnvironmentImpl implements ListNodesInEnvironment {
public class ListNodesInEnvironmentImpl extends BaseListNodesImpl implements ListNodesInEnvironment {
protected final ChefApi api;
protected final ListeningExecutorService userExecutor;
@Resource
@Named(ChefProperties.CHEF_LOGGER)
protected Logger logger = Logger.NULL;
@Inject
ListNodesInEnvironmentImpl(@Named(Constants.PROPERTY_USER_THREADS) ListeningExecutorService userExecutor, ChefApi api) {
this.userExecutor = checkNotNull(userExecutor, "userExecuor");
this.api = checkNotNull(api, "api");
ListNodesInEnvironmentImpl(ChefApi api) {
super(api);
}
@Override
public Iterable<? extends Node> execute(String environmentName) {
return execute(userExecutor, environmentName);
return super.execute(api.listNodesInEnvironment(environmentName));
}
@Override
public Iterable<? extends Node> execute(ListeningExecutorService executor, String environmentName) {
return execute(executor, environmentName, api.listNodesInEnvironment(environmentName));
public Iterable<? extends Node> execute(ExecutorService executor, String environmentName) {
return this.executeConcurrently(MoreExecutors.listeningDecorator(executor), environmentName);
}
private Iterable<? extends Node> execute(final ListeningExecutorService executor, String environmentName, Iterable<String> toGet) {
ListenableFuture<List<Node>> futures = allAsList(transform(toGet, new Function<String, ListenableFuture<Node>>() {
@Override
public ListenableFuture<Node> apply(final String input) {
return executor.submit(new Callable<Node>() {
@Override
public Node call() throws Exception {
return api.getNode(input);
}
});
}
}));
logger.trace(String.format("getting nodes in environment %s: %s", environmentName, Joiner.on(',').join(toGet)));
return getUnchecked(futures);
private Iterable<? extends Node> executeConcurrently(ListeningExecutorService executor,
String environmentName) {
return super.executeConcurrently(executor, api.listNodesInEnvironment(environmentName));
}
}

View File

@ -22,7 +22,11 @@ import static org.testng.Assert.fail;
import java.io.File;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.common.util.concurrent.MoreExecutors;
import org.jclouds.chef.ChefApi;
import org.jclouds.chef.domain.ChecksumStatus;
import org.jclouds.chef.domain.CookbookVersion;
@ -51,6 +55,9 @@ public class ListCookbookVersionsInEnvironmentImplLiveTest extends BaseChefLiveT
private ListCookbookVersionsInEnvironmentImpl strategy;
private CreateNodeAndPopulateAutomaticAttributesImpl creator;
private ExecutorService testExecutorService;
private ListeningExecutorService testListeningExecutorService;
@Override
protected void initialize() {
super.initialize();
@ -63,6 +70,8 @@ public class ListCookbookVersionsInEnvironmentImplLiveTest extends BaseChefLiveT
}
this.strategy = injector.getInstance(ListCookbookVersionsInEnvironmentImpl.class);
this.testExecutorService = Executors.newFixedThreadPool(5);
this.testListeningExecutorService = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(5));
}
@AfterClass(groups = { "integration", "live" })
@ -72,6 +81,10 @@ public class ListCookbookVersionsInEnvironmentImplLiveTest extends BaseChefLiveT
api.deleteCookbook(PREFIX, "1.0.0");
api.deleteCookbook(PREFIX + 1, "0.0.0");
api.deleteCookbook(PREFIX + 1, "1.0.0");
this.testExecutorService.shutdown();
this.testListeningExecutorService.shutdown();
super.tearDown();
}
@ -80,16 +93,52 @@ public class ListCookbookVersionsInEnvironmentImplLiveTest extends BaseChefLiveT
assertTrue(size(strategy.execute("_default")) > 0, "Expected one or more elements");
}
@Test
public void testExecuteConcurrentlyWithExecutorService() {
assertTrue(size(strategy.execute(testExecutorService, "_default")) > 0,
"Expected one or more elements");
}
@Test
public void testExecuteConcurrentlyWithListeningExecutorService() {
assertTrue(size(strategy.execute(testListeningExecutorService, "_default")) > 0,
"Expected one or more elements");
}
@Test
public void testExecuteWithNumVersions() {
assertTrue(size(strategy.execute("_default", "2")) > 0, "Expected one or more elements");
}
@Test
public void testExecuteConcurrentlyWithNumVersionsAndExecutorService() {
assertTrue(size(strategy.execute(testExecutorService, "_default", "2")) > 0,
"Expected one or more elements");
}
@Test
public void testExecuteConcurrentlyWithNumVersionsAndListeningExecutorService() {
assertTrue(size(strategy.execute(testListeningExecutorService, "_default", "2")) > 0,
"Expected one or more elements");
}
@Test
public void testExecuteWithNumVersionsAll() {
assertTrue(size(strategy.execute("_default", "all")) > 0, "Expected one or more elements");
}
@Test
public void testExecuteConcurrentlyWithNumVersionsAllAndExecutorService() {
assertTrue(size(strategy.execute(testExecutorService, "_default", "all")) > 0,
"Expected one or more elements");
}
@Test
public void testExecuteConcurrentlyWithNumVersionsAllAndListeningExecutorService() {
assertTrue(size(strategy.execute(testListeningExecutorService, "_default", "all")) > 0,
"Expected one or more elements");
}
private FilePayload uploadContent(String fileName) throws Exception {
// Define the file you want in the cookbook
File file = new File(System.getProperty("user.dir"), fileName);

View File

@ -19,6 +19,8 @@ package org.jclouds.chef.strategy.internal;
import static com.google.common.collect.Iterables.size;
import static org.testng.Assert.assertTrue;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.common.util.concurrent.MoreExecutors;
import org.jclouds.chef.ChefApi;
import org.jclouds.chef.internal.BaseChefLiveTest;
import org.testng.annotations.AfterClass;
@ -26,6 +28,9 @@ import org.testng.annotations.Test;
import com.google.common.collect.ImmutableSet;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
/**
* Tests behavior of {@code ListNodesImpl} strategies
*/
@ -35,6 +40,9 @@ public class ListNodesImplLiveTest extends BaseChefLiveTest<ChefApi> {
private ListNodesImpl strategy;
private CreateNodeAndPopulateAutomaticAttributesImpl creator;
private ExecutorService testExecutorService;
private ListeningExecutorService testListeningExecutorService;
@Override
protected void initialize() {
super.initialize();
@ -42,6 +50,9 @@ public class ListNodesImplLiveTest extends BaseChefLiveTest<ChefApi> {
this.strategy = injector.getInstance(ListNodesImpl.class);
creator.execute(prefix, ImmutableSet.<String> of());
creator.execute(prefix + 1, ImmutableSet.<String> of());
this.testExecutorService = Executors.newFixedThreadPool(5);
this.testListeningExecutorService = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(5));
}
@AfterClass(groups = { "integration", "live" })
@ -49,6 +60,10 @@ public class ListNodesImplLiveTest extends BaseChefLiveTest<ChefApi> {
protected void tearDown() {
api.deleteNode(prefix);
api.deleteNode(prefix + 1);
this.testExecutorService.shutdown();
this.testListeningExecutorService.shutdown();
super.tearDown();
}
@ -56,4 +71,12 @@ public class ListNodesImplLiveTest extends BaseChefLiveTest<ChefApi> {
public void testExecute() {
assertTrue(size(strategy.execute()) > 0, "Expected one or more elements");
}
public void testExecuteConcurrentlyWithExecutorService() {
assertTrue(size(strategy.execute(testExecutorService)) > 0, "Expected one or more elements");
}
public void testExecuteConcurrentlyWithListeningExecutorService() {
assertTrue(size(strategy.execute(testListeningExecutorService)) > 0, "Expected one or more elements");
}
}

View File

@ -19,6 +19,8 @@ package org.jclouds.chef.strategy.internal;
import static com.google.common.collect.Iterables.size;
import static org.testng.Assert.assertTrue;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.common.util.concurrent.MoreExecutors;
import org.jclouds.chef.ChefApi;
import org.jclouds.chef.internal.BaseChefLiveTest;
import org.testng.annotations.AfterClass;
@ -26,6 +28,9 @@ import org.testng.annotations.Test;
import com.google.common.collect.ImmutableSet;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
/**
* Tests behavior of {@code ListNodesInEnvironmentImpl} strategies
*/
@ -35,13 +40,19 @@ public class ListNodesInEnvironmentImplLiveTest extends BaseChefLiveTest<ChefApi
private ListNodesInEnvironmentImpl strategy;
private CreateNodeAndPopulateAutomaticAttributesImpl creator;
private ExecutorService testExecutorService;
private ListeningExecutorService testListeningExecutorService;
@Override
protected void initialize() {
super.initialize();
this.creator = injector.getInstance(CreateNodeAndPopulateAutomaticAttributesImpl.class);
this.strategy = injector.getInstance(ListNodesInEnvironmentImpl.class);
creator.execute(prefix, ImmutableSet.<String> of());
creator.execute(prefix + 1, ImmutableSet.<String> of());
creator.execute(prefix, ImmutableSet.<String>of());
creator.execute(prefix + 1, ImmutableSet.<String>of());
this.testExecutorService = Executors.newFixedThreadPool(5);
this.testListeningExecutorService = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(5));
}
@AfterClass(groups = { "integration", "live" })
@ -49,6 +60,10 @@ public class ListNodesInEnvironmentImplLiveTest extends BaseChefLiveTest<ChefApi
protected void tearDown() {
api.deleteNode(prefix);
api.deleteNode(prefix + 1);
this.testExecutorService.shutdown();
this.testListeningExecutorService.shutdown();
super.tearDown();
}
@ -56,4 +71,16 @@ public class ListNodesInEnvironmentImplLiveTest extends BaseChefLiveTest<ChefApi
public void testExecute() {
assertTrue(size(strategy.execute("_default")) > 0, "Expected one or more elements");
}
@Test
public void testExecuteConcurrentlyWithExecutorService() {
assertTrue(size(strategy.execute(testExecutorService, "_default")) > 0,
"Expected one or more elements");
}
@Test
public void testExecuteConcurrentlyWithListeningExecutorService() {
assertTrue(size(strategy.execute(testListeningExecutorService, "_default")) > 0,
"Expected one or more elements");
}
}