From 413d64f60fb035991328542a3c94a4c1ed0ea06f Mon Sep 17 00:00:00 2001 From: David Ribeiro Alves Date: Thu, 7 Mar 2013 23:40:33 -0600 Subject: [PATCH] gce-compute - - addressed reviewer changes - increased test coverage - addressed resource leak - updated README --- labs/google-compute/README.txt | 56 ++- labs/google-compute/pom.xml | 15 +- .../GoogleComputeApiMetadata.java | 37 +- .../googlecompute/GoogleComputeConstants.java | 25 +- .../compute/GoogleComputeService.java | 183 ++++++++ .../compute/GoogleComputeServiceAdapter.java | 252 +++++++++++ .../GoogleComputeServiceContextModule.java | 196 +++++++++ .../functions/BuildInstanceMetadata.java | 50 +++ .../functions/GoogleComputeImageToImage.java | 83 ++++ .../functions/InstanceToNodeMetadata.java | 114 +++++ .../functions/MachineTypeToHardware.java | 60 +++ .../OrphanedGroupsFromDeadNodes.java | 61 +++ .../compute/functions/ZoneToLocation.java | 49 +++ .../options/GoogleComputeTemplateOptions.java | 280 ++++++++++++ .../predicates/AllNodesInGroupTerminated.java | 53 +++ ...sWithGroupEncodedIntoNameThenAddToSet.java | 200 +++++++++ ...faultLoginCredentialsForImageStrategy.java | 74 ++++ ...odeCredentialsButOverrideFromTemplate.java | 42 ++ .../config/GoogleComputeRestClientModule.java | 6 + .../googlecompute/domain/Instance.java | 8 +- .../GoogleComputeServiceExpectTest.java | 415 ++++++++++++++++++ .../compute/GoogleComputeServiceLiveTest.java | 96 ++++ .../GoogleComputeImageToImageTest.java | 66 +++ .../OrphanedGroupsFromDeadNodesTest.java | 141 ++++++ .../features/NetworkApiExpectTest.java | 2 +- .../features/ZoneApiExpectTest.java | 16 +- .../internal/BaseGoogleComputeExpectTest.java | 55 ++- ...GoogleComputeServiceContextExpectTest.java | 54 +++ .../BaseGoogleComputeServiceExpectTest.java | 34 ++ .../googlecompute/parse/ParseNetworkTest.java | 4 +- .../parse/ParseZoneListTest.java | 6 +- .../googlecompute/parse/ParseZoneTest.java | 6 +- .../src/test/resources/network_get.json | 4 +- .../src/test/resources/network_insert.json | 2 +- .../src/test/resources/network_list.json | 4 +- .../src/test/resources/zone_get.json | 6 +- .../src/test/resources/zone_list.json | 12 +- 37 files changed, 2702 insertions(+), 65 deletions(-) create mode 100644 labs/google-compute/src/main/java/org/jclouds/googlecompute/compute/GoogleComputeService.java create mode 100644 labs/google-compute/src/main/java/org/jclouds/googlecompute/compute/GoogleComputeServiceAdapter.java create mode 100644 labs/google-compute/src/main/java/org/jclouds/googlecompute/compute/config/GoogleComputeServiceContextModule.java create mode 100644 labs/google-compute/src/main/java/org/jclouds/googlecompute/compute/functions/BuildInstanceMetadata.java create mode 100644 labs/google-compute/src/main/java/org/jclouds/googlecompute/compute/functions/GoogleComputeImageToImage.java create mode 100644 labs/google-compute/src/main/java/org/jclouds/googlecompute/compute/functions/InstanceToNodeMetadata.java create mode 100644 labs/google-compute/src/main/java/org/jclouds/googlecompute/compute/functions/MachineTypeToHardware.java create mode 100644 labs/google-compute/src/main/java/org/jclouds/googlecompute/compute/functions/OrphanedGroupsFromDeadNodes.java create mode 100644 labs/google-compute/src/main/java/org/jclouds/googlecompute/compute/functions/ZoneToLocation.java create mode 100644 labs/google-compute/src/main/java/org/jclouds/googlecompute/compute/options/GoogleComputeTemplateOptions.java create mode 100644 labs/google-compute/src/main/java/org/jclouds/googlecompute/compute/predicates/AllNodesInGroupTerminated.java create mode 100644 labs/google-compute/src/main/java/org/jclouds/googlecompute/compute/strategy/ApplyTemplateOptionsCreateNodesWithGroupEncodedIntoNameThenAddToSet.java create mode 100644 labs/google-compute/src/main/java/org/jclouds/googlecompute/compute/strategy/GoogleComputePopulateDefaultLoginCredentialsForImageStrategy.java create mode 100644 labs/google-compute/src/main/java/org/jclouds/googlecompute/compute/strategy/UseNodeCredentialsButOverrideFromTemplate.java create mode 100644 labs/google-compute/src/test/java/org/jclouds/googlecompute/compute/GoogleComputeServiceExpectTest.java create mode 100644 labs/google-compute/src/test/java/org/jclouds/googlecompute/compute/GoogleComputeServiceLiveTest.java create mode 100644 labs/google-compute/src/test/java/org/jclouds/googlecompute/compute/functions/GoogleComputeImageToImageTest.java create mode 100644 labs/google-compute/src/test/java/org/jclouds/googlecompute/compute/functions/OrphanedGroupsFromDeadNodesTest.java create mode 100644 labs/google-compute/src/test/java/org/jclouds/googlecompute/internal/BaseGoogleComputeServiceContextExpectTest.java create mode 100644 labs/google-compute/src/test/java/org/jclouds/googlecompute/internal/BaseGoogleComputeServiceExpectTest.java diff --git a/labs/google-compute/README.txt b/labs/google-compute/README.txt index 5499eaea76..8c50447559 100644 --- a/labs/google-compute/README.txt +++ b/labs/google-compute/README.txt @@ -1,16 +1,54 @@ -Status: +jclouds Google Compute Engine Provider +====== -All the private apis are implemented and tested. -Snapshots are disabled because they are also disabled in GCE. -Just missing jcloud.compute glue code (coming soon!) -How to run the live tests: +Authenticating into the instances: +-------- -Pre-requisites: -A Google Api account with Google Compute Engine enabled -The access pk (provided by google in PKCS12 format) in pem format. +User: +If no user is provided in GoogleComputeTemplateOptions when launching an instance by defaul "jclouds" is used. -running all tests: +Credential: + +GCE uses exclusively ssh keys to login into instances. +In order for an instance to be sshable a public key must be installed. Public keys are installed if they are present in the project or instance's metatada. + +For an instance to be ssable one of the following must happen: +1 - the project's metadata has an adequately built "sshKeys" entry and a corresponding private key is provided in GoogleComputeTemplateOptions when createNodesInGroup is called. +2 - an instance of GoogleComputeTemplateOptions with an adequate public and private key is provided. + +NOTE: if methods 2 is chosen the global project keys will not be installed in the instance. + +Please refer to Google's documentation on how to form valid project wide ssh keys metadata entries. + +FAQ: +-------- + +* Q. What is the identity for GCE? + +A. the identity is the developer email which can be obtained from the admin GUI. Its usually something in the form: @developer.gserviceaccount.com + +* Q. What is the crendential for GCE + +A. the credential is a private key, in pem format. It can be extracted from the p12 keystore that is obtained when creating a "Service Account" (in the GUI: Google apis console > Api Access > Create another client ID > "Service Account" + +* Q. How to convert a p12 keystore into a pem format jclouds-gce can handle: + +A. + +1. Convert the p12 file into pem format (it will ask for the keystore password, which is usually "notasecret"): + openssl pkcs12 -in .p12 -out .pem -nodes + +2. Extract only the pk and remove passphrase + openssl rsa -in .pem -out .pem + +The last file (.pem) should contain the pk that needs to be passed to google-compute as a string literal property named "google-compute.credential". + + +Running the live tests: +-------- mvn clean install -Plive -Dtest.google-compute.identity=@developer.gserviceaccount.com -Dtest.google-compute.credential= + + diff --git a/labs/google-compute/pom.xml b/labs/google-compute/pom.xml index f2d54bddf6..f085bad08d 100644 --- a/labs/google-compute/pom.xml +++ b/labs/google-compute/pom.xml @@ -63,6 +63,13 @@ jclouds-compute ${jclouds.version} + + org.jclouds + jclouds-compute + ${jclouds.version} + test-jar + test + org.jclouds jclouds-core @@ -77,7 +84,13 @@ test - ch.qos.logback + org.jclouds.driver + jclouds-sshj + ${project.version} + test + + + ch.qos.logback logback-classic test diff --git a/labs/google-compute/src/main/java/org/jclouds/googlecompute/GoogleComputeApiMetadata.java b/labs/google-compute/src/main/java/org/jclouds/googlecompute/GoogleComputeApiMetadata.java index b01193e19a..1c2285dfeb 100644 --- a/labs/google-compute/src/main/java/org/jclouds/googlecompute/GoogleComputeApiMetadata.java +++ b/labs/google-compute/src/main/java/org/jclouds/googlecompute/GoogleComputeApiMetadata.java @@ -18,14 +18,12 @@ */ package org.jclouds.googlecompute; -import static org.jclouds.Constants.PROPERTY_SESSION_INTERVAL; -import static org.jclouds.oauth.v2.config.OAuthProperties.AUDIENCE; -import static org.jclouds.oauth.v2.config.OAuthProperties.SIGNATURE_OR_MAC_ALGORITHM; - -import java.net.URI; -import java.util.Properties; - +import com.google.common.collect.ImmutableSet; +import com.google.common.reflect.TypeToken; +import com.google.inject.Module; import org.jclouds.apis.ApiMetadata; +import org.jclouds.compute.ComputeServiceContext; +import org.jclouds.googlecompute.compute.config.GoogleComputeServiceContextModule; import org.jclouds.googlecompute.config.GoogleComputeParserModule; import org.jclouds.googlecompute.config.GoogleComputeRestClientModule; import org.jclouds.googlecompute.config.OAuthModuleWithoutTypeAdapters; @@ -33,9 +31,14 @@ import org.jclouds.oauth.v2.config.OAuthAuthenticationModule; import org.jclouds.rest.RestContext; import org.jclouds.rest.internal.BaseRestApiMetadata; -import com.google.common.collect.ImmutableSet; -import com.google.common.reflect.TypeToken; -import com.google.inject.Module; +import java.net.URI; +import java.util.Properties; + +import static org.jclouds.Constants.PROPERTY_SESSION_INTERVAL; +import static org.jclouds.compute.config.ComputeServiceProperties.TEMPLATE; +import static org.jclouds.oauth.v2.config.OAuthProperties.AUDIENCE; +import static org.jclouds.oauth.v2.config.OAuthProperties.SIGNATURE_OR_MAC_ALGORITHM; +import static org.jclouds.reflect.Reflection2.typeToken; /** * Implementation of {@link ApiMetadata} for GoogleCompute v1beta13 API @@ -44,9 +47,8 @@ import com.google.inject.Module; */ public class GoogleComputeApiMetadata extends BaseRestApiMetadata { - public static final TypeToken> CONTEXT_TOKEN = new TypeToken>() { - private static final long serialVersionUID = 1L; - }; + public static final TypeToken> CONTEXT_TOKEN = new + TypeToken>() {}; @Override public Builder toBuilder() { @@ -67,6 +69,10 @@ public class GoogleComputeApiMetadata extends BaseRestApiMetadata { properties.put(AUDIENCE, "https://accounts.google.com/o/oauth2/token"); properties.put(SIGNATURE_OR_MAC_ALGORITHM, "RS256"); properties.put(PROPERTY_SESSION_INTERVAL, 3600); + properties.setProperty(TEMPLATE, "osFamily=GCEL,osVersionMatches=1[012].[01][04],locationId=us-central1-a," + + "loginUser=jclouds"); + properties.put(GoogleComputeConstants.OPERATION_COMPLETE_INTERVAL, 500); + properties.put(GoogleComputeConstants.OPERATION_COMPLETE_TIMEOUT, 600000); return properties; } @@ -82,11 +88,14 @@ public class GoogleComputeApiMetadata extends BaseRestApiMetadata { .version("v1beta13") .defaultEndpoint("https://www.googleapis.com/compute/v1beta13") .defaultProperties(GoogleComputeApiMetadata.defaultProperties()) + .view(typeToken(ComputeServiceContext.class)) .defaultModules(ImmutableSet.>builder() .add(GoogleComputeRestClientModule.class) .add(GoogleComputeParserModule.class) .add(OAuthAuthenticationModule.class) - .add(OAuthModuleWithoutTypeAdapters.class).build()); + .add(OAuthModuleWithoutTypeAdapters.class) + .add(GoogleComputeServiceContextModule.class) + .build()); } @Override diff --git a/labs/google-compute/src/main/java/org/jclouds/googlecompute/GoogleComputeConstants.java b/labs/google-compute/src/main/java/org/jclouds/googlecompute/GoogleComputeConstants.java index 2334ee9ce6..c2d8b72d5b 100644 --- a/labs/google-compute/src/main/java/org/jclouds/googlecompute/GoogleComputeConstants.java +++ b/labs/google-compute/src/main/java/org/jclouds/googlecompute/GoogleComputeConstants.java @@ -19,21 +19,40 @@ package org.jclouds.googlecompute; +import com.google.common.annotations.Beta; +import org.jclouds.domain.Location; +import org.jclouds.domain.LocationBuilder; +import org.jclouds.domain.LocationScope; + /** * @author David Alves */ public interface GoogleComputeConstants { + public static final String GOOGLE_PROVIDER_NAME = "google-compute"; + + /** + * The name of the project that keeps public resources. + */ + public static final String GOOGLE_PROJECT = "google"; + public static final String COMPUTE_SCOPE = "https://www.googleapis.com/auth/compute"; public static final String COMPUTE_READONLY_SCOPE = "https://www.googleapis.com/auth/compute.readonly"; /** - * TODO storage scopes should be moved to the GCS api + * The total time, in msecs, to wait for an operation to complete. */ + @Beta + public static final String OPERATION_COMPLETE_TIMEOUT = "jclouds.google-compute.operation-complete-timeout"; - public static final String STORAGE_WRITEONLY_SCOPE = "https://www.googleapis.com/auth/devstorage.write_only"; + /** + * The interval, in msecs, between calls to check whether an operation has completed. + */ + @Beta + public static final String OPERATION_COMPLETE_INTERVAL = "jclouds.google-compute.operation-complete-interval"; - public static final String STORAGE_READONLY_SCOPE = "https://www.googleapis.com/auth/devstorage.read_only"; + public static final Location GOOGLE_PROVIDER_LOCATION = new LocationBuilder().scope(LocationScope.PROVIDER).id + (GOOGLE_PROVIDER_NAME).description(GOOGLE_PROVIDER_NAME).build(); } diff --git a/labs/google-compute/src/main/java/org/jclouds/googlecompute/compute/GoogleComputeService.java b/labs/google-compute/src/main/java/org/jclouds/googlecompute/compute/GoogleComputeService.java new file mode 100644 index 0000000000..556222e492 --- /dev/null +++ b/labs/google-compute/src/main/java/org/jclouds/googlecompute/compute/GoogleComputeService.java @@ -0,0 +1,183 @@ +/* + * Licensed to jclouds, Inc. (jclouds) under one or more + * contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. jclouds licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.jclouds.googlecompute.compute; + +import com.google.common.base.Function; +import com.google.common.base.Optional; +import com.google.common.base.Predicate; +import com.google.common.base.Supplier; +import com.google.common.util.concurrent.ListeningExecutorService; +import org.jclouds.Constants; +import org.jclouds.collect.Memoized; +import org.jclouds.compute.ComputeServiceContext; +import org.jclouds.compute.callables.RunScriptOnNode; +import org.jclouds.compute.domain.Hardware; +import org.jclouds.compute.domain.Image; +import org.jclouds.compute.domain.NodeMetadata; +import org.jclouds.compute.domain.TemplateBuilder; +import org.jclouds.compute.extensions.ImageExtension; +import org.jclouds.compute.functions.GroupNamingConvention; +import org.jclouds.compute.internal.BaseComputeService; +import org.jclouds.compute.internal.PersistNodeCredentials; +import org.jclouds.compute.options.TemplateOptions; +import org.jclouds.compute.reference.ComputeServiceConstants; +import org.jclouds.compute.strategy.CreateNodesInGroupThenAddToSet; +import org.jclouds.compute.strategy.DestroyNodeStrategy; +import org.jclouds.compute.strategy.GetImageStrategy; +import org.jclouds.compute.strategy.GetNodeMetadataStrategy; +import org.jclouds.compute.strategy.InitializeRunScriptOnNodeOrPlaceInBadMap; +import org.jclouds.compute.strategy.ListNodesStrategy; +import org.jclouds.compute.strategy.RebootNodeStrategy; +import org.jclouds.compute.strategy.ResumeNodeStrategy; +import org.jclouds.compute.strategy.SuspendNodeStrategy; +import org.jclouds.domain.Credentials; +import org.jclouds.domain.Location; +import org.jclouds.googlecompute.GoogleComputeApi; +import org.jclouds.googlecompute.compute.options.GoogleComputeTemplateOptions; +import org.jclouds.googlecompute.config.UserProject; +import org.jclouds.googlecompute.domain.Operation; +import org.jclouds.http.HttpResponse; +import org.jclouds.scriptbuilder.functions.InitAdminAccess; + +import javax.inject.Inject; +import javax.inject.Named; +import javax.inject.Provider; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicReference; + +import static com.google.common.base.Preconditions.checkNotNull; +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static org.jclouds.compute.config.ComputeServiceProperties.TIMEOUT_NODE_RUNNING; +import static org.jclouds.compute.config.ComputeServiceProperties.TIMEOUT_NODE_SUSPENDED; +import static org.jclouds.compute.config.ComputeServiceProperties.TIMEOUT_NODE_TERMINATED; +import static org.jclouds.googlecompute.GoogleComputeConstants.OPERATION_COMPLETE_INTERVAL; +import static org.jclouds.googlecompute.GoogleComputeConstants.OPERATION_COMPLETE_TIMEOUT; +import static org.jclouds.util.Predicates2.retry; + +/** + * @author David Alves + */ +public class GoogleComputeService extends BaseComputeService { + + private final Function, Set> findOrphanedGroups; + private final GroupNamingConvention.Factory namingConvention; + private final GoogleComputeApi api; + private final Supplier project; + private final Predicate> operationDonePredicate; + private final long operationCompleteCheckInterval; + private final long operationCompleteCheckTimeout; + + @Inject + protected GoogleComputeService(ComputeServiceContext context, + Map credentialStore, + @Memoized Supplier> images, + @Memoized Supplier> hardwareProfiles, + @Memoized Supplier> locations, + ListNodesStrategy listNodesStrategy, + GetImageStrategy getImageStrategy, + GetNodeMetadataStrategy getNodeMetadataStrategy, + CreateNodesInGroupThenAddToSet runNodesAndAddToSetStrategy, + RebootNodeStrategy rebootNodeStrategy, + DestroyNodeStrategy destroyNodeStrategy, + ResumeNodeStrategy resumeNodeStrategy, + SuspendNodeStrategy suspendNodeStrategy, + Provider templateBuilderProvider, + @Named("DEFAULT") Provider templateOptionsProvider, + @Named(TIMEOUT_NODE_RUNNING) Predicate> nodeRunning, + @Named(TIMEOUT_NODE_TERMINATED) Predicate> + nodeTerminated, + @Named(TIMEOUT_NODE_SUSPENDED) + Predicate> nodeSuspended, + InitializeRunScriptOnNodeOrPlaceInBadMap.Factory initScriptRunnerFactory, + InitAdminAccess initAdminAccess, + RunScriptOnNode.Factory runScriptOnNodeFactory, + PersistNodeCredentials persistNodeCredentials, + ComputeServiceConstants.Timeouts timeouts, + @Named(Constants.PROPERTY_USER_THREADS) ListeningExecutorService userExecutor, + Optional imageExtension, + Function, Set> findOrphanedGroups, + GroupNamingConvention.Factory namingConvention, + GoogleComputeApi api, + @UserProject Supplier project, + Predicate> operationDonePredicate, + @Named(OPERATION_COMPLETE_INTERVAL) Long operationCompleteCheckInterval, + @Named(OPERATION_COMPLETE_TIMEOUT) Long operationCompleteCheckTimeout) { + + super(context, credentialStore, images, hardwareProfiles, locations, listNodesStrategy, getImageStrategy, + getNodeMetadataStrategy, runNodesAndAddToSetStrategy, rebootNodeStrategy, destroyNodeStrategy, + resumeNodeStrategy, suspendNodeStrategy, templateBuilderProvider, templateOptionsProvider, nodeRunning, + nodeTerminated, nodeSuspended, initScriptRunnerFactory, initAdminAccess, runScriptOnNodeFactory, + persistNodeCredentials, timeouts, userExecutor, imageExtension); + this.findOrphanedGroups = checkNotNull(findOrphanedGroups, "find orphaned groups function"); + this.namingConvention = checkNotNull(namingConvention, "naming convention factory"); + this.api = checkNotNull(api, "google compute api"); + this.project = checkNotNull(project, "user project name"); + this.operationDonePredicate = checkNotNull(operationDonePredicate, "operation completed predicate"); + this.operationCompleteCheckInterval = checkNotNull(operationCompleteCheckInterval, + "operation completed check interval"); + this.operationCompleteCheckTimeout = checkNotNull(operationCompleteCheckTimeout, + "operation completed check timeout"); + } + + @Override + protected synchronized void cleanUpIncidentalResourcesOfDeadNodes(Set deadNodes) { + Set orphanedGroups = findOrphanedGroups.apply(deadNodes); + for (String orphanedGroup : orphanedGroups) { + cleanUpNetworksAndFirewallsForGroup(orphanedGroup); + } + } + + + protected void cleanUpNetworksAndFirewallsForGroup(String groupName) { + String resourceName = namingConvention.create().sharedNameForGroup(groupName); + AtomicReference operation = new AtomicReference(api.getFirewallApiForProject(project.get()) + .delete(resourceName)); + + retry(operationDonePredicate, operationCompleteCheckTimeout, operationCompleteCheckInterval, + MILLISECONDS).apply(operation); + + if (operation.get().getHttpError().isPresent()) { + HttpResponse response = operation.get().getHttpError().get(); + logger.warn("delete orphaned firewall failed. Http Error Code: " + response.getStatusCode() + + " HttpError: " + response.getMessage()); + } + + operation = new AtomicReference(api.getNetworkApiForProject(project.get()).delete(resourceName)); + + retry(operationDonePredicate, operationCompleteCheckTimeout, operationCompleteCheckInterval, + MILLISECONDS).apply(operation); + + if (operation.get().getHttpError().isPresent()) { + HttpResponse response = operation.get().getHttpError().get(); + logger.warn("delete orphaned network failed. Http Error Code: " + response.getStatusCode() + + " HttpError: " + response.getMessage()); + } + } + + + /** + * returns template options, except of type {@link GoogleComputeTemplateOptions}. + */ + @Override + public GoogleComputeTemplateOptions templateOptions() { + return GoogleComputeTemplateOptions.class.cast(super.templateOptions()); + } +} diff --git a/labs/google-compute/src/main/java/org/jclouds/googlecompute/compute/GoogleComputeServiceAdapter.java b/labs/google-compute/src/main/java/org/jclouds/googlecompute/compute/GoogleComputeServiceAdapter.java new file mode 100644 index 0000000000..ab083b0a4f --- /dev/null +++ b/labs/google-compute/src/main/java/org/jclouds/googlecompute/compute/GoogleComputeServiceAdapter.java @@ -0,0 +1,252 @@ +/* + * Licensed to jclouds, Inc. (jclouds) under one or more + * contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. jclouds licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.jclouds.googlecompute.compute; + +import com.google.common.base.Function; +import com.google.common.base.Objects; +import com.google.common.base.Predicate; +import com.google.common.base.Supplier; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.util.concurrent.UncheckedTimeoutException; +import com.google.inject.Inject; +import org.jclouds.compute.ComputeServiceAdapter; +import org.jclouds.compute.domain.Hardware; +import org.jclouds.compute.domain.Template; +import org.jclouds.compute.options.TemplateOptions; +import org.jclouds.compute.reference.ComputeServiceConstants; +import org.jclouds.domain.LoginCredentials; +import org.jclouds.googlecompute.GoogleComputeApi; +import org.jclouds.googlecompute.compute.options.GoogleComputeTemplateOptions; +import org.jclouds.googlecompute.config.UserProject; +import org.jclouds.googlecompute.domain.Image; +import org.jclouds.googlecompute.domain.Instance; +import org.jclouds.googlecompute.domain.InstanceTemplate; +import org.jclouds.googlecompute.domain.MachineType; +import org.jclouds.googlecompute.domain.Operation; +import org.jclouds.googlecompute.domain.Zone; +import org.jclouds.http.HttpResponse; +import org.jclouds.logging.Logger; + +import javax.annotation.Resource; +import javax.inject.Named; +import java.net.URI; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; + +import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.base.Preconditions.checkState; +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static org.jclouds.googlecompute.GoogleComputeConstants.GOOGLE_PROJECT; +import static org.jclouds.googlecompute.GoogleComputeConstants.OPERATION_COMPLETE_INTERVAL; +import static org.jclouds.googlecompute.GoogleComputeConstants.OPERATION_COMPLETE_TIMEOUT; +import static org.jclouds.googlecompute.domain.Instance.NetworkInterface.AccessConfig.Type; +import static org.jclouds.util.Predicates2.retry; + +/** + * @author David Alves + */ +public class GoogleComputeServiceAdapter implements ComputeServiceAdapter { + + @Resource + @Named(ComputeServiceConstants.COMPUTE_LOGGER) + protected Logger logger = Logger.NULL; + + private final GoogleComputeApi api; + private final Supplier userProject; + private final Function> metatadaFromTemplateOptions; + private final Predicate> retryOperationDonePredicate; + private final long operationCompleteCheckInterval; + private final long operationCompleteCheckTimeout; + + @Inject + public GoogleComputeServiceAdapter(GoogleComputeApi api, + @UserProject Supplier userProject, + Function> metatadaFromTemplateOptions, + Predicate> operationDonePredicate, + @Named(OPERATION_COMPLETE_INTERVAL) Long operationCompleteCheckInterval, + @Named(OPERATION_COMPLETE_TIMEOUT) Long operationCompleteCheckTimeout) { + this.api = checkNotNull(api, "google compute api"); + this.userProject = checkNotNull(userProject, "user project name"); + this.metatadaFromTemplateOptions = checkNotNull(metatadaFromTemplateOptions, + "metadata from template options function"); + this.operationCompleteCheckInterval = checkNotNull(operationCompleteCheckInterval, + "operation completed check interval"); + this.operationCompleteCheckTimeout = checkNotNull(operationCompleteCheckTimeout, + "operation completed check timeout"); + this.retryOperationDonePredicate = retry(operationDonePredicate, operationCompleteCheckTimeout, + operationCompleteCheckInterval, TimeUnit.MILLISECONDS); + } + + @Override + public NodeAndInitialCredentials createNodeWithGroupEncodedIntoName( + final String group, final String name, Template template) { + + checkNotNull(template, "template"); + + GoogleComputeTemplateOptions options = GoogleComputeTemplateOptions.class.cast(template.getOptions()).clone(); + checkState(options.getNetwork().isPresent(), "network was not present in template options"); + Hardware hardware = checkNotNull(template.getHardware(), "hardware must be set"); + URI machineType = checkNotNull(hardware.getUri(), "hardware uri must be set"); + + InstanceTemplate instanceTemplate = InstanceTemplate.builder() + .forMachineType(machineType); + + if (options.isEnableNat()) { + instanceTemplate.addNetworkInterface(options.getNetwork().get(), Type.ONE_TO_ONE_NAT); + } else { + instanceTemplate.addNetworkInterface(options.getNetwork().get()); + } + + LoginCredentials credentials = getFromImageAndOverrideIfRequired(template.getImage(), options); + + ImmutableMap.Builder metadataBuilder = metatadaFromTemplateOptions.apply(options); + instanceTemplate.metadata(metadataBuilder.build()); + instanceTemplate.tags(options.getTags()); + instanceTemplate.serviceAccounts(options.getServiceAccounts()); + instanceTemplate.image(checkNotNull(template.getImage().getUri(), "image URI is null")); + + Operation operation = api.getInstanceApiForProject(userProject.get()) + .createInZone(name, instanceTemplate, template.getLocation().getId()); + + if (options.shouldBlockUntilRunning()) { + waitOperationDone(operation); + } + + // some times the newly created instances are not immediately returned + AtomicReference instance = new AtomicReference(); + + retry(new Predicate>() { + @Override + public boolean apply(AtomicReference input) { + input.set(api.getInstanceApiForProject(userProject.get()).get(name)); + return input.get() != null; + } + }, operationCompleteCheckTimeout, operationCompleteCheckInterval, MILLISECONDS).apply(instance); + + return new NodeAndInitialCredentials(instance.get(), name, credentials); + } + + + @Override + public Iterable listHardwareProfiles() { + return api.getMachineTypeApiForProject(userProject.get()).list().concat(); + } + + @Override + public Iterable listImages() { + return ImmutableSet.builder() + .addAll(api.getImageApiForProject(userProject.get()).list().concat()) + .addAll(api.getImageApiForProject(GOOGLE_PROJECT).list().concat()) + .build(); + } + + @Override + public Image getImage(String id) { + return Objects.firstNonNull(api.getImageApiForProject(userProject.get()).get(id), + api.getImageApiForProject(GOOGLE_PROJECT).get(id)); + } + + @Override + public Iterable listLocations() { + return api.getZoneApiForProject(userProject.get()).list().concat(); + } + + @Override + public Instance getNode(String name) { + return api.getInstanceApiForProject(userProject.get()).get(name); + } + + @Override + public Iterable listNodes() { + return api.getInstanceApiForProject(userProject.get()).list().concat(); + } + + @Override + public void destroyNode(final String name) { + waitOperationDone(api.getInstanceApiForProject(userProject.get()).delete(name)); + } + + @Override + public void rebootNode(String name) { + throw new UnsupportedOperationException("reboot is not supported by GCE"); + } + + @Override + public void resumeNode(String name) { + throw new UnsupportedOperationException("resume is not supported by GCE"); + } + + @Override + public void suspendNode(String name) { + throw new UnsupportedOperationException("suspend is not supported by GCE"); + } + + private LoginCredentials getFromImageAndOverrideIfRequired(org.jclouds.compute.domain.Image image, + GoogleComputeTemplateOptions options) { + LoginCredentials defaultCredentials = image.getDefaultCredentials(); + String[] keys = defaultCredentials.getPrivateKey().split(":"); + String publicKey = keys[0]; + String privateKey = keys[1]; + + LoginCredentials.Builder credentialsBuilder = defaultCredentials.toBuilder(); + credentialsBuilder.privateKey(privateKey); + + // LoginCredentials from image stores the public key along with the private key in the privateKey field + // @see GoogleComputePopulateDefaultLoginCredentialsForImageStrategy + // so if options doesn't have a public key set we set it from the default + if (options.getPublicKey() == null) { + options.authorizePublicKey(publicKey); + } + if (options.hasLoginPrivateKeyOption()) { + credentialsBuilder.privateKey(options.getPrivateKey()); + } + if (options.getLoginUser() != null) { + credentialsBuilder.identity(options.getLoginUser()); + } + if (options.hasLoginPasswordOption()) { + credentialsBuilder.password(options.getLoginPassword()); + } + if (options.shouldAuthenticateSudo() != null) { + credentialsBuilder.authenticateSudo(options.shouldAuthenticateSudo()); + } + LoginCredentials credentials = credentialsBuilder.build(); + options.overrideLoginCredentials(credentials); + return credentials; + } + + private void waitOperationDone(Operation operation) { + AtomicReference operationRef = new AtomicReference(operation); + + // wait for the operation to complete + if (!retryOperationDonePredicate.apply(operationRef)) { + throw new UncheckedTimeoutException("operation did not reach DONE state" + operationRef.get()); + } + + // check if the operation failed + if (operationRef.get().getHttpError().isPresent()) { + HttpResponse response = operationRef.get().getHttpError().get(); + throw new IllegalStateException("operation failed. Http Error Code: " + response.getStatusCode() + + " HttpError: " + response.getMessage()); + } + } + +} diff --git a/labs/google-compute/src/main/java/org/jclouds/googlecompute/compute/config/GoogleComputeServiceContextModule.java b/labs/google-compute/src/main/java/org/jclouds/googlecompute/compute/config/GoogleComputeServiceContextModule.java new file mode 100644 index 0000000000..b0421b3c85 --- /dev/null +++ b/labs/google-compute/src/main/java/org/jclouds/googlecompute/compute/config/GoogleComputeServiceContextModule.java @@ -0,0 +1,196 @@ +/* + * Licensed to jclouds, Inc. (jclouds) under one or more + * contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. jclouds licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.jclouds.googlecompute.compute.config; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Function; +import com.google.common.base.Optional; +import com.google.common.base.Predicate; +import com.google.common.base.Supplier; +import com.google.common.collect.ImmutableMap; +import com.google.inject.Injector; +import com.google.inject.Provides; +import com.google.inject.TypeLiteral; +import org.jclouds.collect.Memoized; +import org.jclouds.compute.ComputeService; +import org.jclouds.compute.ComputeServiceAdapter; +import org.jclouds.compute.config.ComputeServiceAdapterContextModule; +import org.jclouds.compute.domain.Hardware; +import org.jclouds.compute.domain.NodeMetadata; +import org.jclouds.compute.extensions.ImageExtension; +import org.jclouds.compute.options.TemplateOptions; +import org.jclouds.compute.strategy.PopulateDefaultLoginCredentialsForImageStrategy; +import org.jclouds.compute.strategy.PrioritizeCredentialsFromTemplate; +import org.jclouds.compute.strategy.impl.CreateNodesWithGroupEncodedIntoNameThenAddToSet; +import org.jclouds.domain.Location; +import org.jclouds.googlecompute.GoogleComputeApi; +import org.jclouds.googlecompute.compute.GoogleComputeService; +import org.jclouds.googlecompute.compute.GoogleComputeServiceAdapter; +import org.jclouds.googlecompute.compute.functions.BuildInstanceMetadata; +import org.jclouds.googlecompute.compute.functions.GoogleComputeImageToImage; +import org.jclouds.googlecompute.compute.functions.InstanceToNodeMetadata; +import org.jclouds.googlecompute.compute.functions.MachineTypeToHardware; +import org.jclouds.googlecompute.compute.functions.OrphanedGroupsFromDeadNodes; +import org.jclouds.googlecompute.compute.functions.ZoneToLocation; +import org.jclouds.googlecompute.compute.options.GoogleComputeTemplateOptions; +import org.jclouds.googlecompute.compute.predicates.AllNodesInGroupTerminated; +import org.jclouds.googlecompute.compute.strategy.ApplyTemplateOptionsCreateNodesWithGroupEncodedIntoNameThenAddToSet; +import org.jclouds.googlecompute.compute.strategy.GoogleComputePopulateDefaultLoginCredentialsForImageStrategy; +import org.jclouds.googlecompute.compute.strategy.UseNodeCredentialsButOverrideFromTemplate; +import org.jclouds.googlecompute.config.UserProject; +import org.jclouds.googlecompute.domain.Image; +import org.jclouds.googlecompute.domain.Instance; +import org.jclouds.googlecompute.domain.MachineType; +import org.jclouds.googlecompute.domain.Zone; + +import javax.inject.Singleton; +import java.net.URI; +import java.util.Map; +import java.util.Set; + +import static com.google.common.collect.Iterables.transform; +import static com.google.common.collect.Maps.uniqueIndex; + +/** + * @author David Alves + */ +public class GoogleComputeServiceContextModule + extends ComputeServiceAdapterContextModule { + + @Override + protected void configure() { + super.configure(); + + bind(ComputeService.class).to(GoogleComputeService.class); + + bind(new TypeLiteral>() {}) + .to(GoogleComputeServiceAdapter.class); + + bind(new TypeLiteral>() {}) + .to(InstanceToNodeMetadata.class); + + bind(new TypeLiteral>() {}) + .to(MachineTypeToHardware.class); + + bind(new TypeLiteral>() {}) + .to(GoogleComputeImageToImage.class); + + bind(new TypeLiteral>() {}) + .to(ZoneToLocation.class); + + bind(new TypeLiteral>>() {}) + .to(BuildInstanceMetadata.class); + + bind(PopulateDefaultLoginCredentialsForImageStrategy.class) + .to(GoogleComputePopulateDefaultLoginCredentialsForImageStrategy.class); + + bind(CreateNodesWithGroupEncodedIntoNameThenAddToSet.class).to( + ApplyTemplateOptionsCreateNodesWithGroupEncodedIntoNameThenAddToSet.class); + + bind(TemplateOptions.class).to(GoogleComputeTemplateOptions.class); + + bind(new TypeLiteral, Set>>() {}) + .to(OrphanedGroupsFromDeadNodes.class); + + bind(new TypeLiteral>() {}).to(AllNodesInGroupTerminated.class); + + bind(PrioritizeCredentialsFromTemplate.class).to(UseNodeCredentialsButOverrideFromTemplate.class); + + install(new LocationsFromComputeServiceAdapterModule() {}); + + } + + @Provides + @Singleton + @Memoized + public Supplier> provideImagesMap( + final Supplier> images) { + return new Supplier>() { + @Override + public Map get() { + return uniqueIndex(images.get(), new Function() { + @Override + public URI apply(org.jclouds.compute.domain.Image input) { + return input.getUri(); + } + }); + } + }; + } + + @Provides + @Singleton + @Memoized + public Supplier> provideHardwaresMap( + final Supplier> hardwares) { + return new Supplier>() { + @Override + public Map get() { + return uniqueIndex(hardwares.get(), new Function() { + @Override + public URI apply(Hardware input) { + return input.getUri(); + } + }); + } + }; + } + + @Provides + @Singleton + @Memoized + public Supplier> provideLocations( + final GoogleComputeApi api, final Function zoneToLocation, + final @UserProject Supplier userProject) { + return new Supplier>() { + @Override + public Map get() { + return uniqueIndex(transform(api.getZoneApiForProject(userProject.get()).list().concat(), zoneToLocation), + new Function() { + @Override + public URI apply(Location input) { + return (URI) input.getMetadata().get("selfLink"); + } + }); + } + }; + } + + @Override + protected Optional provideImageExtension(Injector i) { + return Optional.absent(); + } + + @VisibleForTesting + public static final Map toPortableNodeStatus = + ImmutableMap.builder() + .put(Instance.Status.PROVISIONING, NodeMetadata.Status.PENDING) + .put(Instance.Status.STAGING, NodeMetadata.Status.PENDING) + .put(Instance.Status.RUNNING, NodeMetadata.Status.RUNNING) + .put(Instance.Status.STOPPING, NodeMetadata.Status.PENDING) + .put(Instance.Status.STOPPED, NodeMetadata.Status.SUSPENDED) + .put(Instance.Status.TERMINATED, NodeMetadata.Status.TERMINATED).build(); + + @Singleton + @Provides + protected Map toPortableNodeStatus() { + return toPortableNodeStatus; + } +} diff --git a/labs/google-compute/src/main/java/org/jclouds/googlecompute/compute/functions/BuildInstanceMetadata.java b/labs/google-compute/src/main/java/org/jclouds/googlecompute/compute/functions/BuildInstanceMetadata.java new file mode 100644 index 0000000000..e867369b93 --- /dev/null +++ b/labs/google-compute/src/main/java/org/jclouds/googlecompute/compute/functions/BuildInstanceMetadata.java @@ -0,0 +1,50 @@ +/* + * Licensed to jclouds, Inc. (jclouds) under one or more + * contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. jclouds licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.jclouds.googlecompute.compute.functions; + +import com.google.common.base.Function; +import com.google.common.collect.ImmutableMap; +import org.jclouds.compute.options.TemplateOptions; + +import javax.inject.Singleton; + +import static com.google.common.base.Preconditions.checkNotNull; +import static java.lang.String.format; + +/** + * Prepares metadata from the provided TemplateOptions + * + * @author David Alves + */ +@Singleton +public class BuildInstanceMetadata implements Function> { + + @Override + public ImmutableMap.Builder apply(TemplateOptions input) { + ImmutableMap.Builder builder = ImmutableMap.builder(); + if (input.getPublicKey() != null) { + builder.put("sshKeys", format("%s:%s %s@localhost", checkNotNull(input.getLoginUser(), + "loginUser cannot be null"), input.getPublicKey(), input.getLoginUser())); + } + builder.putAll(input.getUserMetadata()); + return builder; + } + +} diff --git a/labs/google-compute/src/main/java/org/jclouds/googlecompute/compute/functions/GoogleComputeImageToImage.java b/labs/google-compute/src/main/java/org/jclouds/googlecompute/compute/functions/GoogleComputeImageToImage.java new file mode 100644 index 0000000000..dd781cbd4f --- /dev/null +++ b/labs/google-compute/src/main/java/org/jclouds/googlecompute/compute/functions/GoogleComputeImageToImage.java @@ -0,0 +1,83 @@ +/* + * Licensed to jclouds, Inc. (jclouds) under one or more + * contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. jclouds licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.jclouds.googlecompute.compute.functions; + +import com.google.common.base.Function; +import com.google.common.collect.Lists; +import org.jclouds.compute.domain.ImageBuilder; +import org.jclouds.compute.domain.OperatingSystem; +import org.jclouds.compute.domain.OsFamily; +import org.jclouds.googlecompute.GoogleComputeConstants; +import org.jclouds.googlecompute.domain.Image; + +import java.util.List; + +import static com.google.common.base.Joiner.on; +import static com.google.common.collect.Iterables.getLast; +import static com.google.common.collect.Iterables.limit; +import static com.google.common.collect.Iterables.skip; +import static org.jclouds.compute.domain.Image.Status; +import static org.jclouds.googlecompute.GoogleComputeConstants.GOOGLE_PROVIDER_LOCATION; + +/** + * Transforms a google compute domain specific image to a generic Image object. + * + * @author David Alves + */ +public class GoogleComputeImageToImage implements Function { + + + @Override + public org.jclouds.compute.domain.Image apply(Image image) { + ImageBuilder builder = new ImageBuilder() + .id(image.getName()) + .name(image.getName()) + .providerId(image.getId()) + .description(image.getDescription().orNull()) + .status(Status.AVAILABLE) + .location(GOOGLE_PROVIDER_LOCATION) + .uri(image.getSelfLink()); + + List splits = Lists.newArrayList(image.getName().split("-")); + OperatingSystem.Builder osBuilder = defaultOperatingSystem(image); + if (splits == null || splits.size() == 0 || splits.size() < 3) { + return builder.operatingSystem(osBuilder.build()).build(); + } + + OsFamily family = OsFamily.fromValue(splits.get(0)); + if (family != OsFamily.UNRECOGNIZED) { + osBuilder.family(family); + } + + String version = on(".").join(limit(skip(splits, 1), splits.size() - 2)); + osBuilder.version(version); + + builder.version(getLast(splits)); + return builder.operatingSystem(osBuilder.build()).build(); + } + + private OperatingSystem.Builder defaultOperatingSystem(Image image) { + return OperatingSystem.builder() + .family(OsFamily.LINUX) + .is64Bit(true) + .description(image.getName()); + } + +} diff --git a/labs/google-compute/src/main/java/org/jclouds/googlecompute/compute/functions/InstanceToNodeMetadata.java b/labs/google-compute/src/main/java/org/jclouds/googlecompute/compute/functions/InstanceToNodeMetadata.java new file mode 100644 index 0000000000..583430987e --- /dev/null +++ b/labs/google-compute/src/main/java/org/jclouds/googlecompute/compute/functions/InstanceToNodeMetadata.java @@ -0,0 +1,114 @@ +/* + * Licensed to jclouds, Inc. (jclouds) under one or more + * contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. jclouds licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.jclouds.googlecompute.compute.functions; + +import com.google.common.base.Function; +import com.google.common.base.Supplier; +import com.google.common.collect.ImmutableSet; +import org.jclouds.collect.Memoized; +import org.jclouds.compute.domain.Hardware; +import org.jclouds.compute.domain.Image; +import org.jclouds.compute.domain.NodeMetadata; +import org.jclouds.compute.domain.NodeMetadataBuilder; +import org.jclouds.compute.functions.GroupNamingConvention; +import org.jclouds.domain.Location; +import org.jclouds.googlecompute.domain.Instance; + +import javax.inject.Inject; +import java.net.URI; +import java.util.Map; +import java.util.Set; + +import static com.google.common.base.Preconditions.checkNotNull; + +/** + * Transforms a google compute domain Instance into a generic NodeMetatada object. + * + * @author David Alves + */ +public class InstanceToNodeMetadata implements Function { + + private final Map toPortableNodeStatus; + private final GroupNamingConvention nodeNamingConvention; + private final Supplier> images; + private final Supplier> hardwares; + private final Supplier> locations; + + @Inject + public InstanceToNodeMetadata(Map toPortableNodeStatus, + GroupNamingConvention.Factory namingConvention, + @Memoized Supplier> images, + @Memoized Supplier> hardwares, + @Memoized Supplier> locations) { + this.toPortableNodeStatus = toPortableNodeStatus; + this.nodeNamingConvention = namingConvention.createWithoutPrefix(); + this.images = images; + this.hardwares = hardwares; + this.locations = locations; + } + + @Override + public NodeMetadata apply(Instance input) { + Map imagesMap = images.get(); + Image image = checkNotNull(imagesMap.get(checkNotNull(input.getImage(), "image")), + "no image for %s. images: %s", input.getImage(), imagesMap.values()); + + return new NodeMetadataBuilder() + .id(input.getName()) + .name(input.getName()) + .providerId(input.getId()) + .hostname(input.getName()) + .imageId(image.getId()) + .location(checkNotNull(locations.get().get(input.getZone()), "location for %s", input.getZone())) + .hardware(checkNotNull(hardwares.get().get(input.getMachineType()), "hardware type for %s", + input.getMachineType().toString())) + .operatingSystem(image.getOperatingSystem()) + .status(toPortableNodeStatus.get(input.getStatus())) + .tags(input.getTags()) + .uri(input.getSelfLink()) + .userMetadata(input.getMetadata()) + .group(nodeNamingConvention.groupInUniqueNameOrNull(input.getName())) + .privateAddresses(collectPrivateAddresses(input)) + .publicAddresses(collectPublicAddresses(input)) + .build(); + } + + private Set collectPrivateAddresses(Instance input) { + ImmutableSet.Builder privateAddressesBuilder = ImmutableSet.builder(); + for (Instance.NetworkInterface networkInterface : input.getNetworkInterfaces()) { + if (networkInterface.getNetworkIP().isPresent()) { + privateAddressesBuilder.add(networkInterface.getNetworkIP().get()); + } + } + return privateAddressesBuilder.build(); + } + + private Set collectPublicAddresses(Instance input) { + ImmutableSet.Builder publicAddressesBuilder = ImmutableSet.builder(); + for (Instance.NetworkInterface networkInterface : input.getNetworkInterfaces()) { + for (Instance.NetworkInterface.AccessConfig accessConfig : networkInterface.getAccessConfigs()) { + if (accessConfig.getNatIP().isPresent()) { + publicAddressesBuilder.add(accessConfig.getNatIP().get()); + } + } + } + return publicAddressesBuilder.build(); + } +} diff --git a/labs/google-compute/src/main/java/org/jclouds/googlecompute/compute/functions/MachineTypeToHardware.java b/labs/google-compute/src/main/java/org/jclouds/googlecompute/compute/functions/MachineTypeToHardware.java new file mode 100644 index 0000000000..33d23fa634 --- /dev/null +++ b/labs/google-compute/src/main/java/org/jclouds/googlecompute/compute/functions/MachineTypeToHardware.java @@ -0,0 +1,60 @@ +/* + * Licensed to jclouds, Inc. (jclouds) under one or more + * contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. jclouds licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.jclouds.googlecompute.compute.functions; + +import com.google.common.base.Function; +import com.google.common.collect.ImmutableSet; +import org.jclouds.compute.domain.Hardware; +import org.jclouds.compute.domain.HardwareBuilder; +import org.jclouds.compute.domain.Processor; +import org.jclouds.compute.domain.Volume; +import org.jclouds.compute.domain.internal.VolumeImpl; +import org.jclouds.googlecompute.domain.MachineType; + +/** + * Transforms a google compute domain specific machine type to a generic Hardware object. + * + * @author David Alves + */ +public class MachineTypeToHardware implements Function { + + @Override + public Hardware apply(MachineType input) { + return new HardwareBuilder() + .id(input.getName()) + .name(input.getName()) + .hypervisor("kvm") + .processor(new Processor(input.getGuestCpus(), 1.0)) + .providerId(input.getId()) + .ram(input.getMemoryMb()) + .uri(input.getSelfLink()) + .volumes(collectVolumes(input)) + .build(); + } + + private Iterable collectVolumes(MachineType input) { + ImmutableSet.Builder volumes = ImmutableSet.builder(); + for (MachineType.EphemeralDisk disk : input.getEphemeralDisks()) { + volumes.add(new VolumeImpl(null, Volume.Type.LOCAL, new Integer(disk.getDiskGb()).floatValue(), null, true, + false)); + } + return volumes.build(); + } +} diff --git a/labs/google-compute/src/main/java/org/jclouds/googlecompute/compute/functions/OrphanedGroupsFromDeadNodes.java b/labs/google-compute/src/main/java/org/jclouds/googlecompute/compute/functions/OrphanedGroupsFromDeadNodes.java new file mode 100644 index 0000000000..f60397aa56 --- /dev/null +++ b/labs/google-compute/src/main/java/org/jclouds/googlecompute/compute/functions/OrphanedGroupsFromDeadNodes.java @@ -0,0 +1,61 @@ +/* + * Licensed to jclouds, Inc. (jclouds) under one or more + * contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. jclouds licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.jclouds.googlecompute.compute.functions; + +import com.google.common.base.Function; +import com.google.common.base.Predicate; +import com.google.common.collect.Sets; +import org.jclouds.compute.domain.NodeMetadata; + +import javax.inject.Inject; +import javax.inject.Singleton; +import java.util.Set; + +/** + * @author David Alves + */ +@Singleton +public class OrphanedGroupsFromDeadNodes implements Function, Set> { + + private final Predicate isOrphanedGroupPredicate; + + @Inject + public OrphanedGroupsFromDeadNodes(Predicate isOrphanedGroupPredicate) { + this.isOrphanedGroupPredicate = isOrphanedGroupPredicate; + } + + + @Override + public Set apply(Set deadNodes) { + Set groups = Sets.newLinkedHashSet(); + for (NodeMetadata deadNode : deadNodes) { + groups.add(deadNode.getGroup()); + } + Set orphanedGroups = Sets.newLinkedHashSet(); + for (String group : groups) { + if (isOrphanedGroupPredicate.apply(group)) { + orphanedGroups.add(group); + } + } + return orphanedGroups; + } + + +} diff --git a/labs/google-compute/src/main/java/org/jclouds/googlecompute/compute/functions/ZoneToLocation.java b/labs/google-compute/src/main/java/org/jclouds/googlecompute/compute/functions/ZoneToLocation.java new file mode 100644 index 0000000000..d6a8763c94 --- /dev/null +++ b/labs/google-compute/src/main/java/org/jclouds/googlecompute/compute/functions/ZoneToLocation.java @@ -0,0 +1,49 @@ +/* + * Licensed to jclouds, Inc. (jclouds) under one or more + * contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. jclouds licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.jclouds.googlecompute.compute.functions; + +import com.google.common.base.Function; +import com.google.common.collect.ImmutableMap; +import org.jclouds.domain.Location; +import org.jclouds.domain.LocationBuilder; +import org.jclouds.domain.LocationScope; +import org.jclouds.googlecompute.domain.Zone; + +import static com.google.common.base.Preconditions.checkNotNull; +import static org.jclouds.googlecompute.GoogleComputeConstants.GOOGLE_PROVIDER_LOCATION; + +/** + * Transforms a google compute domain specific zone to a generic Zone object. + * + * @author David Alves + */ +public class ZoneToLocation implements Function { + + @Override + public Location apply(Zone input) { + return new LocationBuilder() + .description(input.getDescription().orNull()) + .metadata(ImmutableMap.of("selfLink", (Object) checkNotNull(input.getSelfLink(), "zone URI"))) + .id(input.getName()) + .scope(LocationScope.ZONE) + .parent(GOOGLE_PROVIDER_LOCATION) + .build(); + } +} diff --git a/labs/google-compute/src/main/java/org/jclouds/googlecompute/compute/options/GoogleComputeTemplateOptions.java b/labs/google-compute/src/main/java/org/jclouds/googlecompute/compute/options/GoogleComputeTemplateOptions.java new file mode 100644 index 0000000000..9f8d9c98ad --- /dev/null +++ b/labs/google-compute/src/main/java/org/jclouds/googlecompute/compute/options/GoogleComputeTemplateOptions.java @@ -0,0 +1,280 @@ +package org.jclouds.googlecompute.compute.options; + +import com.google.common.base.Optional; +import com.google.common.collect.Sets; +import org.jclouds.compute.options.TemplateOptions; +import org.jclouds.domain.LoginCredentials; +import org.jclouds.googlecompute.domain.Instance; +import org.jclouds.scriptbuilder.domain.Statement; + +import java.net.URI; +import java.util.Map; +import java.util.Set; + +import static com.google.common.base.Optional.fromNullable; +import static org.jclouds.googlecompute.domain.Instance.ServiceAccount; + +/** + * Instance options specific to Google Compute Engine. + * + * @author David Alves + */ +public class GoogleComputeTemplateOptions extends TemplateOptions { + + private Optional network = Optional.absent(); + private Optional networkName = Optional.absent(); + private Set serviceAccounts = Sets.newLinkedHashSet(); + private boolean enableNat = true; + + @Override + public GoogleComputeTemplateOptions clone() { + GoogleComputeTemplateOptions options = new GoogleComputeTemplateOptions(); + copyTo(options); + return options; + } + + @Override + public void copyTo(TemplateOptions to) { + super.copyTo(to); + if (to instanceof GoogleComputeTemplateOptions) { + GoogleComputeTemplateOptions eTo = GoogleComputeTemplateOptions.class.cast(to); + eTo.network(getNetwork().orNull()); + eTo.network(getNetworkName().orNull()); + eTo.serviceAccounts(getServiceAccounts()); + eTo.enableNat(isEnableNat()); + } + } + + /** + * @see #getNetworkName() + */ + public GoogleComputeTemplateOptions network(String networkName) { + this.networkName = fromNullable(networkName); + return this; + } + + /** + * @see #getNetwork() + */ + public GoogleComputeTemplateOptions network(URI network) { + this.network = fromNullable(network); + return this; + } + + /** + * @see #getServiceAccounts() + * @see ServiceAccount + */ + public GoogleComputeTemplateOptions addServiceAccount(ServiceAccount serviceAccout) { + this.serviceAccounts.add(serviceAccout); + return this; + } + + /** + * @see #getServiceAccounts() + * @see ServiceAccount + */ + public GoogleComputeTemplateOptions serviceAccounts(Set serviceAccounts) { + this.serviceAccounts = Sets.newLinkedHashSet(serviceAccounts); + return this; + } + + /** + * @see #isEnableNat() + */ + public GoogleComputeTemplateOptions enableNat(boolean enableNat) { + this.enableNat = enableNat; + return this; + } + + /** + * {@inheritDoc} + */ + @Override + public GoogleComputeTemplateOptions blockOnPort(int port, int seconds) { + return GoogleComputeTemplateOptions.class.cast(super.blockOnPort(port, seconds)); + } + + /** + * {@inheritDoc} + */ + @Override + public GoogleComputeTemplateOptions inboundPorts(int... ports) { + return GoogleComputeTemplateOptions.class.cast(super.inboundPorts(ports)); + } + + /** + * {@inheritDoc} + */ + @Override + public GoogleComputeTemplateOptions authorizePublicKey(String publicKey) { + return GoogleComputeTemplateOptions.class.cast(super.authorizePublicKey(publicKey)); + } + + /** + * {@inheritDoc} + */ + @Override + public GoogleComputeTemplateOptions installPrivateKey(String privateKey) { + return GoogleComputeTemplateOptions.class.cast(super.installPrivateKey(privateKey)); + } + + /** + * {@inheritDoc} + */ + @Override + public GoogleComputeTemplateOptions blockUntilRunning(boolean blockUntilRunning) { + return GoogleComputeTemplateOptions.class.cast(super.blockUntilRunning(blockUntilRunning)); + } + + /** + * {@inheritDoc} + */ + @Override + public GoogleComputeTemplateOptions dontAuthorizePublicKey() { + return GoogleComputeTemplateOptions.class.cast(super.dontAuthorizePublicKey()); + } + + /** + * {@inheritDoc} + */ + @Override + public GoogleComputeTemplateOptions nameTask(String name) { + return GoogleComputeTemplateOptions.class.cast(super.nameTask(name)); + } + + /** + * {@inheritDoc} + */ + @Override + public GoogleComputeTemplateOptions runAsRoot(boolean runAsRoot) { + return GoogleComputeTemplateOptions.class.cast(super.runAsRoot(runAsRoot)); + } + + /** + * {@inheritDoc} + */ + @Override + public GoogleComputeTemplateOptions runScript(Statement script) { + return GoogleComputeTemplateOptions.class.cast(super.runScript(script)); + } + + /** + * {@inheritDoc} + */ + @Override + public GoogleComputeTemplateOptions overrideLoginCredentials(LoginCredentials overridingCredentials) { + return GoogleComputeTemplateOptions.class.cast(super.overrideLoginCredentials(overridingCredentials)); + } + + /** + * {@inheritDoc} + */ + @Override + public GoogleComputeTemplateOptions overrideLoginPassword(String password) { + return GoogleComputeTemplateOptions.class.cast(super.overrideLoginPassword(password)); + } + + /** + * {@inheritDoc} + */ + @Override + public GoogleComputeTemplateOptions overrideLoginPrivateKey(String privateKey) { + return GoogleComputeTemplateOptions.class.cast(super.overrideLoginPrivateKey(privateKey)); + } + + /** + * {@inheritDoc} + */ + @Override + public GoogleComputeTemplateOptions overrideLoginUser(String loginUser) { + return GoogleComputeTemplateOptions.class.cast(super.overrideLoginUser(loginUser)); + } + + /** + * {@inheritDoc} + */ + @Override + public GoogleComputeTemplateOptions overrideAuthenticateSudo(boolean authenticateSudo) { + return GoogleComputeTemplateOptions.class.cast(super.overrideAuthenticateSudo(authenticateSudo)); + } + + /** + * {@inheritDoc} + */ + @Override + public GoogleComputeTemplateOptions userMetadata(Map userMetadata) { + return GoogleComputeTemplateOptions.class.cast(super.userMetadata(userMetadata)); + } + + /** + * {@inheritDoc} + */ + @Override + public GoogleComputeTemplateOptions userMetadata(String key, String value) { + return GoogleComputeTemplateOptions.class.cast(super.userMetadata(key, value)); + } + + /** + * {@inheritDoc} + */ + @Override + public GoogleComputeTemplateOptions tags(Iterable tags) { + return GoogleComputeTemplateOptions.class.cast(super.tags(tags)); + } + + /** + * {@inheritDoc} + */ + @Override + public GoogleComputeTemplateOptions wrapInInitScript(boolean wrapInInitScript) { + return GoogleComputeTemplateOptions.class.cast(super.wrapInInitScript(wrapInInitScript)); + } + + /** + * {@inheritDoc} + */ + @Override + public GoogleComputeTemplateOptions runScript(String script) { + return GoogleComputeTemplateOptions.class.cast(super.runScript(script)); + } + + /** + * {@inheritDoc} + */ + @Override + public GoogleComputeTemplateOptions blockOnComplete(boolean blockOnComplete) { + return GoogleComputeTemplateOptions.class.cast(super.blockOnComplete(blockOnComplete)); + } + + /** + * @return the ServiceAccounts to enable in the instances. + */ + public Set getServiceAccounts() { + return serviceAccounts; + } + + /** + * @return the URI of an existing network the instances will be attached to. If no network URI or network name are + * provided a new network will be created for the project. + */ + public Optional getNetwork() { + return network; + } + + /** + * @return the name of an existing network the instances will be attached to, the network is assumed to belong to + * user's project. If no network URI network name are provided a new network will be created for the project. + */ + public Optional getNetworkName() { + return networkName; + } + + /** + * @return whether an AccessConfig with Type ONE_TO_ONE_NAT should be enabled in the instances. When true + * instances will have a NAT address that will be publicly accessible. + */ + public boolean isEnableNat() { + return enableNat; + } +} diff --git a/labs/google-compute/src/main/java/org/jclouds/googlecompute/compute/predicates/AllNodesInGroupTerminated.java b/labs/google-compute/src/main/java/org/jclouds/googlecompute/compute/predicates/AllNodesInGroupTerminated.java new file mode 100644 index 0000000000..b03f26b001 --- /dev/null +++ b/labs/google-compute/src/main/java/org/jclouds/googlecompute/compute/predicates/AllNodesInGroupTerminated.java @@ -0,0 +1,53 @@ +/* + * Licensed to jclouds, Inc. (jclouds) under one or more + * contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. jclouds licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.jclouds.googlecompute.compute.predicates; + +import com.google.common.base.Predicate; +import org.jclouds.compute.ComputeService; + +import javax.inject.Inject; +import javax.inject.Singleton; + +import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.collect.Iterables.all; +import static com.google.common.collect.Sets.filter; +import static org.jclouds.compute.predicates.NodePredicates.TERMINATED; +import static org.jclouds.compute.predicates.NodePredicates.all; +import static org.jclouds.compute.predicates.NodePredicates.inGroup; + +/** + * @author David Alves + */ +@Singleton +public class AllNodesInGroupTerminated implements Predicate { + + private final ComputeService computeService; + + @Inject + public AllNodesInGroupTerminated(ComputeService computeService) { + this.computeService = checkNotNull(computeService, "compute service"); + } + + + @Override + public boolean apply(String groupName) { + return all(filter(computeService.listNodesDetailsMatching(all()), inGroup(groupName)), TERMINATED); + } +} diff --git a/labs/google-compute/src/main/java/org/jclouds/googlecompute/compute/strategy/ApplyTemplateOptionsCreateNodesWithGroupEncodedIntoNameThenAddToSet.java b/labs/google-compute/src/main/java/org/jclouds/googlecompute/compute/strategy/ApplyTemplateOptionsCreateNodesWithGroupEncodedIntoNameThenAddToSet.java new file mode 100644 index 0000000000..8645ea9f22 --- /dev/null +++ b/labs/google-compute/src/main/java/org/jclouds/googlecompute/compute/strategy/ApplyTemplateOptionsCreateNodesWithGroupEncodedIntoNameThenAddToSet.java @@ -0,0 +1,200 @@ +/* + * Licensed to jclouds, Inc. (jclouds) under one or more + * contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. jclouds licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.jclouds.googlecompute.compute.strategy; + +import com.google.common.base.Predicate; +import com.google.common.base.Supplier; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Multimap; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningExecutorService; +import org.jclouds.Constants; +import org.jclouds.compute.config.CustomizationResponse; +import org.jclouds.compute.domain.NodeMetadata; +import org.jclouds.compute.domain.Template; +import org.jclouds.compute.functions.GroupNamingConvention; +import org.jclouds.compute.strategy.CreateNodeWithGroupEncodedIntoName; +import org.jclouds.compute.strategy.CustomizeNodeAndAddToGoodMapOrPutExceptionIntoBadMap; +import org.jclouds.compute.strategy.ListNodesStrategy; +import org.jclouds.compute.strategy.impl.CreateNodesWithGroupEncodedIntoNameThenAddToSet; +import org.jclouds.googlecompute.GoogleComputeApi; +import org.jclouds.googlecompute.compute.options.GoogleComputeTemplateOptions; +import org.jclouds.googlecompute.config.UserProject; +import org.jclouds.googlecompute.domain.Firewall; +import org.jclouds.googlecompute.domain.Network; +import org.jclouds.googlecompute.domain.Operation; +import org.jclouds.googlecompute.options.FirewallOptions; + +import javax.inject.Inject; +import javax.inject.Named; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicReference; + +import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.collect.ImmutableSet.of; +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static org.jclouds.googlecompute.GoogleComputeConstants.OPERATION_COMPLETE_INTERVAL; +import static org.jclouds.googlecompute.GoogleComputeConstants.OPERATION_COMPLETE_TIMEOUT; +import static org.jclouds.util.Predicates2.retry; + +/** + * @author David Alves + */ +public class ApplyTemplateOptionsCreateNodesWithGroupEncodedIntoNameThenAddToSet extends + CreateNodesWithGroupEncodedIntoNameThenAddToSet { + + public static final String EXTERIOR_RANGE = "0.0.0.0/0"; + public static final String DEFAULT_INTERNAL_NETWORK_RANGE = "10.0.0.0/8"; + + private final GoogleComputeApi api; + private final Supplier userProject; + private final Predicate> operationDonePredicate; + private final long operationCompleteCheckInterval; + private final long operationCompleteCheckTimeout; + + @Inject + protected ApplyTemplateOptionsCreateNodesWithGroupEncodedIntoNameThenAddToSet( + CreateNodeWithGroupEncodedIntoName addNodeWithGroupStrategy, + ListNodesStrategy listNodesStrategy, + GroupNamingConvention.Factory namingConvention, + @Named(Constants.PROPERTY_USER_THREADS) + ListeningExecutorService userExecutor, + CustomizeNodeAndAddToGoodMapOrPutExceptionIntoBadMap.Factory + customizeNodeAndAddToGoodMapOrPutExceptionIntoBadMapFactory, + GoogleComputeApi api, + @UserProject Supplier userProject, + Predicate> operationDonePredicate, + @Named(OPERATION_COMPLETE_INTERVAL) Long operationCompleteCheckInterval, + @Named(OPERATION_COMPLETE_TIMEOUT) Long operationCompleteCheckTimeout) { + super(addNodeWithGroupStrategy, listNodesStrategy, namingConvention, userExecutor, + customizeNodeAndAddToGoodMapOrPutExceptionIntoBadMapFactory); + + this.api = checkNotNull(api, "google compute api"); + this.userProject = checkNotNull(userProject, "user project name"); + this.operationCompleteCheckInterval = checkNotNull(operationCompleteCheckInterval, + "operation completed check interval"); + this.operationCompleteCheckTimeout = checkNotNull(operationCompleteCheckTimeout, + "operation completed check timeout"); + this.operationDonePredicate = operationDonePredicate; + } + + @Override + public Map> execute(String group, int count, Template template, + Set goodNodes, Map badNodes, + Multimap customizationResponses) { + + String sharedResourceName = namingConvention.create().sharedNameForGroup(group); + Template mutableTemplate = template.clone(); + GoogleComputeTemplateOptions templateOptions = GoogleComputeTemplateOptions.class.cast(mutableTemplate + .getOptions()); + assert template.getOptions().equals(templateOptions) : "options didn't clone properly"; + + // get or create the network and create a firewall with the users configuration + Network network = getOrCreateNetwork(templateOptions, sharedResourceName); + createFirewall(templateOptions, network, sharedResourceName); + templateOptions.network(network.getSelfLink()); + + return super.execute(group, count, mutableTemplate, goodNodes, badNodes, customizationResponses); + } + + /** + * Try and find a network either previously created by jclouds or user defined. + */ + private Network getOrCreateNetwork(GoogleComputeTemplateOptions templateOptions, String sharedResourceName) { + + String networkName = templateOptions.getNetworkName().isPresent() ? templateOptions.getNetworkName().get() : + sharedResourceName; + + // check if the network was previously created (cache???) + Network network = api.getNetworkApiForProject(userProject.get()).get(networkName); + + if (network != null) { + return network; + } + + if (network == null && templateOptions.getNetwork().isPresent()) { + throw new IllegalStateException("user defined network does not exist: " + templateOptions.getNetwork().get()); + } + + AtomicReference operation = new AtomicReference(api.getNetworkApiForProject(userProject + .get()).createInIPv4Range(sharedResourceName, DEFAULT_INTERNAL_NETWORK_RANGE)); + retry(operationDonePredicate, operationCompleteCheckTimeout, operationCompleteCheckInterval, + MILLISECONDS).apply(operation); + if (operation.get().getHttpError().isPresent()) { + throw new IllegalStateException("Could not create network, operation failed" + operation); + } + + return checkNotNull(api.getNetworkApiForProject(userProject.get()).get(sharedResourceName), + "no network with name %s was found", sharedResourceName); + + } + + /** + * Tries to find if a firewall already exists for this group, if not it creates one. + * + * @see org.jclouds.googlecompute.features.FirewallAsyncApi#patch(String, org.jclouds.googlecompute.options.FirewallOptions) + */ + private void createFirewall(GoogleComputeTemplateOptions templateOptions, Network network, + String sharedResourceName) { + + Firewall firewall = api.getFirewallApiForProject(userProject.get()).get(sharedResourceName); + + if (firewall != null) { + return; + } + + ImmutableSet.Builder rules = ImmutableSet.builder(); + + Firewall.Rule.Builder tcpRule = Firewall.Rule.builder(); + tcpRule.IPProtocol(Firewall.Rule.IPProtocol.TCP); + Firewall.Rule.Builder udpRule = Firewall.Rule.builder(); + udpRule.IPProtocol(Firewall.Rule.IPProtocol.UDP); + for (Integer port : templateOptions.getInboundPorts()) { + tcpRule.addPort(port); + udpRule.addPort(port); + } + rules.add(tcpRule.build()); + rules.add(udpRule.build()); + + + FirewallOptions options = new FirewallOptions() + .name(sharedResourceName) + .network(network.getSelfLink()) + .sourceTags(templateOptions.getTags()) + .allowedRules(rules.build()) + .sourceRanges(of(DEFAULT_INTERNAL_NETWORK_RANGE, EXTERIOR_RANGE)); + + AtomicReference operation = new AtomicReference(api.getFirewallApiForProject(userProject + .get()).createInNetwork( + sharedResourceName, + network.getSelfLink(), + options)); + + retry(operationDonePredicate, operationCompleteCheckTimeout, operationCompleteCheckInterval, + MILLISECONDS).apply(operation); + + if (operation.get().getHttpError().isPresent()) { + throw new IllegalStateException("Could not create firewall, operation failed" + operation.get()); + } + } + + +} diff --git a/labs/google-compute/src/main/java/org/jclouds/googlecompute/compute/strategy/GoogleComputePopulateDefaultLoginCredentialsForImageStrategy.java b/labs/google-compute/src/main/java/org/jclouds/googlecompute/compute/strategy/GoogleComputePopulateDefaultLoginCredentialsForImageStrategy.java new file mode 100644 index 0000000000..4f454e2648 --- /dev/null +++ b/labs/google-compute/src/main/java/org/jclouds/googlecompute/compute/strategy/GoogleComputePopulateDefaultLoginCredentialsForImageStrategy.java @@ -0,0 +1,74 @@ +/* + * Licensed to jclouds, Inc. (jclouds) under one or more + * contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. jclouds licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.jclouds.googlecompute.compute.strategy; + +import com.google.inject.Inject; +import org.jclouds.compute.domain.TemplateBuilderSpec; +import org.jclouds.compute.strategy.PopulateDefaultLoginCredentialsForImageStrategy; +import org.jclouds.domain.LoginCredentials; +import org.jclouds.ssh.internal.RsaSshKeyPairGenerator; + +import javax.annotation.PostConstruct; +import javax.inject.Named; +import javax.inject.Singleton; +import java.security.NoSuchAlgorithmException; +import java.util.Map; + +import static com.google.common.base.Preconditions.checkNotNull; +import static org.jclouds.compute.config.ComputeServiceProperties.TEMPLATE; + +/** + * @author David Alves + */ +@Singleton +public class GoogleComputePopulateDefaultLoginCredentialsForImageStrategy implements + PopulateDefaultLoginCredentialsForImageStrategy { + + private final TemplateBuilderSpec templateBuilder; + private final RsaSshKeyPairGenerator keyPairGenerator; + private String compoundKey; + + @Inject + GoogleComputePopulateDefaultLoginCredentialsForImageStrategy(@Named(TEMPLATE) String templateSpec, + RsaSshKeyPairGenerator keyPairGenerator) + throws NoSuchAlgorithmException { + this.templateBuilder = TemplateBuilderSpec.parse(checkNotNull(templateSpec, "template builder spec")); + checkNotNull(templateBuilder.getLoginUser(), "template builder spec must provide a loginUser"); + this.keyPairGenerator = checkNotNull(keyPairGenerator, "keypair generator"); + } + + @PostConstruct + private void generateKeys() { + Map keys = keyPairGenerator.get(); + // as we need to store both the pubk and the pk, store them separated by : (base64 does not contain that char) + compoundKey = String.format("%s:%s", checkNotNull(keys.get("public"), "public key cannot be null"), + checkNotNull(keys.get("private"), "private key cannot be null")); + } + + @Override + public LoginCredentials apply(Object image) { + return LoginCredentials.builder() + .authenticateSudo(templateBuilder.getAuthenticateSudo() != null ? + templateBuilder.getAuthenticateSudo() : false) + .privateKey(compoundKey) + .user(templateBuilder.getLoginUser()).build(); + } + +} diff --git a/labs/google-compute/src/main/java/org/jclouds/googlecompute/compute/strategy/UseNodeCredentialsButOverrideFromTemplate.java b/labs/google-compute/src/main/java/org/jclouds/googlecompute/compute/strategy/UseNodeCredentialsButOverrideFromTemplate.java new file mode 100644 index 0000000000..0377539421 --- /dev/null +++ b/labs/google-compute/src/main/java/org/jclouds/googlecompute/compute/strategy/UseNodeCredentialsButOverrideFromTemplate.java @@ -0,0 +1,42 @@ +package org.jclouds.googlecompute.compute.strategy; + +import com.google.common.base.Function; +import com.google.inject.Inject; +import com.google.inject.Singleton; +import org.jclouds.compute.domain.Template; +import org.jclouds.compute.options.RunScriptOptions; +import org.jclouds.compute.strategy.PrioritizeCredentialsFromTemplate; +import org.jclouds.domain.LoginCredentials; + +import static com.google.common.base.Preconditions.checkNotNull; + +/** + * GCE needs the credentials to create the node so the node credentials already take the Image credentials into account, + * as such only overriding the TemplateOptions credentials is required. + * + * @author David Alves + */ +@Singleton +public class UseNodeCredentialsButOverrideFromTemplate extends PrioritizeCredentialsFromTemplate { + + + @Inject + public UseNodeCredentialsButOverrideFromTemplate( + Function credentialsFromImageOrTemplateOptions) { + super(credentialsFromImageOrTemplateOptions); + } + + public LoginCredentials apply(Template template, LoginCredentials fromNode) { + RunScriptOptions options = checkNotNull(template.getOptions(), "template options are required"); + LoginCredentials.Builder builder = LoginCredentials.builder(fromNode); + if (options.getLoginUser() != null) + builder.user(template.getOptions().getLoginUser()); + if (options.getLoginPassword() != null) + builder.password(options.getLoginPassword()); + if (options.getLoginPrivateKey() != null) + builder.privateKey(options.getLoginPrivateKey()); + if (options.shouldAuthenticateSudo() != null && options.shouldAuthenticateSudo()) + builder.authenticateSudo(true); + return builder.build(); + } +} diff --git a/labs/google-compute/src/main/java/org/jclouds/googlecompute/config/GoogleComputeRestClientModule.java b/labs/google-compute/src/main/java/org/jclouds/googlecompute/config/GoogleComputeRestClientModule.java index e36eaee0b7..97c7258e7e 100644 --- a/labs/google-compute/src/main/java/org/jclouds/googlecompute/config/GoogleComputeRestClientModule.java +++ b/labs/google-compute/src/main/java/org/jclouds/googlecompute/config/GoogleComputeRestClientModule.java @@ -60,6 +60,7 @@ import org.jclouds.http.annotation.ServerError; import org.jclouds.json.config.GsonModule.DateAdapter; import org.jclouds.json.config.GsonModule.Iso8601DateAdapter; import org.jclouds.location.Provider; +import org.jclouds.location.config.LocationModule; import org.jclouds.rest.ConfiguresRestClient; import org.jclouds.rest.config.RestClientModule; @@ -164,4 +165,9 @@ public class GoogleComputeRestClientModule extends RestClientModule networkIP; private final Set accessConfigs; @ConstructorProperties({ @@ -565,7 +565,7 @@ public class Instance extends Resource { Set accessConfigs) { this.name = checkNotNull(name, "name"); this.network = checkNotNull(network, "network"); - this.networkIP = checkNotNull(networkIP, "networkIP"); + this.networkIP = fromNullable(networkIP); this.accessConfigs = accessConfigs == null ? ImmutableSet.of() : accessConfigs; } @@ -586,7 +586,7 @@ public class Instance extends Resource { /** * @return An IPV4 internal network address to assign to this instance. */ - public String getNetworkIP() { + public Optional getNetworkIP() { return networkIP; } @@ -698,7 +698,7 @@ public class Instance extends Resource { public Builder fromNetworkInterface(NetworkInterface in) { return this.network(in.getNetwork()) - .networkIP(in.getNetworkIP()) + .networkIP(in.getNetworkIP().orNull()) .accessConfigs(in.getAccessConfigs()); } } diff --git a/labs/google-compute/src/test/java/org/jclouds/googlecompute/compute/GoogleComputeServiceExpectTest.java b/labs/google-compute/src/test/java/org/jclouds/googlecompute/compute/GoogleComputeServiceExpectTest.java new file mode 100644 index 0000000000..439f3c1651 --- /dev/null +++ b/labs/google-compute/src/test/java/org/jclouds/googlecompute/compute/GoogleComputeServiceExpectTest.java @@ -0,0 +1,415 @@ +/* + * Licensed to jclouds, Inc. (jclouds) under one or more + * contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. jclouds licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.jclouds.googlecompute.compute; + +import com.google.common.base.Throwables; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import org.jclouds.compute.ComputeService; +import org.jclouds.compute.RunNodesException; +import org.jclouds.compute.domain.Template; +import org.jclouds.domain.Location; +import org.jclouds.googlecompute.compute.options.GoogleComputeTemplateOptions; +import org.jclouds.googlecompute.domain.Instance; +import org.jclouds.googlecompute.features.InstanceApiExpectTest; +import org.jclouds.googlecompute.internal.BaseGoogleComputeServiceExpectTest; +import org.jclouds.http.HttpRequest; +import org.jclouds.http.HttpResponse; +import org.jclouds.util.Strings2; +import org.testng.annotations.Test; + +import javax.ws.rs.core.MediaType; +import java.io.IOException; +import java.util.List; +import java.util.Properties; +import java.util.Set; + +import static com.google.common.collect.Iterables.getOnlyElement; +import static org.jclouds.googlecompute.GoogleComputeConstants.COMPUTE_READONLY_SCOPE; +import static org.jclouds.googlecompute.GoogleComputeConstants.COMPUTE_SCOPE; +import static org.jclouds.googlecompute.features.FirewallApiExpectTest.GET_FIREWALL_REQUEST; +import static org.jclouds.googlecompute.features.ImageApiExpectTest.LIST_PROJECT_IMAGES_REQUEST; +import static org.jclouds.googlecompute.features.ImageApiExpectTest.LIST_PROJECT_IMAGES_RESPONSE; +import static org.jclouds.googlecompute.features.InstanceApiExpectTest.LIST_INSTANCES_REQUEST; +import static org.jclouds.googlecompute.features.InstanceApiExpectTest.LIST_INSTANCES_RESPONSE; +import static org.jclouds.googlecompute.features.MachineTypeApiExpectTest.LIST_MACHINE_TYPES_REQUEST; +import static org.jclouds.googlecompute.features.MachineTypeApiExpectTest.LIST_MACHINE_TYPES_RESPONSE; +import static org.jclouds.googlecompute.features.NetworkApiExpectTest.GET_NETWORK_REQUEST; +import static org.jclouds.googlecompute.features.OperationApiExpectTest.GET_OPERATION_REQUEST; +import static org.jclouds.googlecompute.features.OperationApiExpectTest.GET_OPERATION_RESPONSE; +import static org.jclouds.googlecompute.features.ZoneApiExpectTest.LIST_ZONES_REQ; +import static org.jclouds.googlecompute.features.ZoneApiExpectTest.LIST_ZONES_RESPONSE; +import static org.jclouds.util.Strings2.toStringAndClose; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; + + +/** + * @author David Alves + */ +@Test(groups = "unit") +public class GoogleComputeServiceExpectTest extends BaseGoogleComputeServiceExpectTest { + + public static final HttpRequest LIST_GOOGLE_IMAGES_REQUEST = HttpRequest + .builder() + .method("GET") + .endpoint("https://www.googleapis.com/compute/v1beta13/projects/google/images") + .addHeader("Accept", "application/json") + .addHeader("Authorization", "Bearer " + TOKEN).build(); + + public static final HttpResponse LIST_GOOGLE_IMAGES_RESPONSE = HttpResponse.builder().statusCode(200) + .payload(staticPayloadFromResource("/image_list_single_page.json")).build(); + + private HttpRequest INSERT_NETWORK_REQUEST = HttpRequest + .builder() + .method("POST") + .endpoint("https://www.googleapis.com/compute/v1beta13/projects/myproject/networks") + .addHeader("Accept", "application/json") + .addHeader("Authorization", "Bearer " + TOKEN) + .payload(payloadFromStringWithContentType("{\"name\":\"jclouds-test\",\"IPv4Range\":\"10.0.0.0/8\"}", + MediaType.APPLICATION_JSON)) + .build(); + + private HttpRequest INSERT_FIREWALL_REQUEST = HttpRequest + .builder() + .method("POST") + .endpoint("https://www.googleapis.com/compute/v1beta13/projects/myproject/firewalls") + .addHeader("Accept", "application/json") + .addHeader("Authorization", "Bearer " + TOKEN) + .payload(payloadFromStringWithContentType("{\"name\":\"jclouds-test\",\"network\":\"https://www.googleapis" + + ".com/compute/v1beta13/projects/myproject/networks/jclouds-test\"," + + "\"sourceRanges\":[\"10.0.0.0/8\",\"0.0.0.0/0\"],\"allowed\":[{\"IPProtocol\":\"tcp\"," + + "\"ports\":[\"22\"]}," + + "{\"IPProtocol\":\"udp\",\"ports\":[\"22\"]}]}", + MediaType.APPLICATION_JSON)) + .build(); + + private HttpResponse GET_NETWORK_RESPONSE = HttpResponse.builder().statusCode(200) + .payload(payloadFromStringWithContentType("{\n" + + " \"kind\": \"compute#network\",\n" + + " \"id\": \"13024414170909937976\",\n" + + " \"creationTimestamp\": \"2012-10-24T20:13:19.967\",\n" + + " \"selfLink\": \"https://www.googleapis" + + ".com/compute/v1beta13/projects/myproject/networks/jclouds-test\",\n" + + " \"name\": \"jclouds-test\",\n" + + " \"description\": \"test network\",\n" + + " \"IPv4Range\": \"10.0.0.0/8\",\n" + + " \"gatewayIPv4\": \"10.0.0.1\"\n" + + "}", MediaType.APPLICATION_JSON)).build(); + + private HttpResponse SUCESSFULL_OPERATION_RESPONSE = HttpResponse.builder().statusCode(200) + .payload(payloadFromResource("/operation.json")).build(); + + + private HttpResponse getInstanceResponseForInstanceAndNetworkAndStatus(String instanceName, String networkName, + String status) throws + IOException { + return HttpResponse.builder().statusCode(200) + .payload(payloadFromStringWithContentType( + replaceInstanceNameNetworkAndStatusOnResource("/instance_get.json", + instanceName, networkName, status), + "application/json")).build(); + } + + private HttpResponse getListInstancesResponseForSingleInstanceAndNetworkAndStatus(String instanceName, + String networkName, + String status) { + return HttpResponse.builder().statusCode(200) + .payload(payloadFromStringWithContentType( + replaceInstanceNameNetworkAndStatusOnResource("/instance_list.json", + instanceName, networkName, status), + "application/json")).build(); + } + + private String replaceInstanceNameNetworkAndStatusOnResource(String resourceName, String instanceName, + String networkName, String status) { + try { + return Strings2.toStringAndClose(this.getClass().getResourceAsStream(resourceName)).replace("test-0", + instanceName).replace("default", networkName).replace("RUNNING", status); + } catch (IOException e) { + throw Throwables.propagate(e); + } + } + + private HttpRequest createInstanceRequestForInstance(String instanceName, String networkName, String publicKey) { + return HttpRequest + .builder() + .method("POST") + .endpoint("https://www.googleapis.com/compute/v1beta13/projects/myproject/instances") + .addHeader("Accept", "application/json") + .addHeader("Authorization", "Bearer " + TOKEN) + .payload(payloadFromStringWithContentType("{\"name\":\"" + instanceName + "\"," + + "\"machineType\":\"https://www.googleapis" + + ".com/compute/v1beta13/projects/myproject/machineTypes/n1-standard-1\"," + + "\"zone\":\"https://www.googleapis" + + ".com/compute/v1beta13/projects/myproject/zones/us-central1-a\"," + + "\"image\":\"https://www.googleapis" + + ".com/compute/v1beta13/projects/google/images/gcel-12-04-v20121106\"," + + "\"tags\":[],\"serviceAccounts\":[]," + + "\"networkInterfaces\":[{\"network\":\"https://www.googleapis" + + ".com/compute/v1beta13/projects/myproject/networks/" + networkName + "\"," + + "\"accessConfigs\":[{\"type\":\"ONE_TO_ONE_NAT\"}]}]," + + "\"metadata\":{\"kind\":\"compute#metadata\",\"items\":[{\"key\":\"sshKeys\"," + + "\"value\":\"jclouds:" + + publicKey + " jclouds@localhost\"}]}}", + MediaType.APPLICATION_JSON)).build(); + } + + private HttpRequest getInstanceRequestForInstance(String instanceName) { + return HttpRequest + .builder() + .method("GET") + .endpoint("https://www.googleapis" + + ".com/compute/v1beta13/projects/myproject/instances/" + instanceName) + .addHeader("Accept", "application/json") + .addHeader("Authorization", "Bearer " + TOKEN).build(); + } + + + @Override + protected Properties setupProperties() { + Properties overrides = super.setupProperties(); + overrides.put("google-compute.identity", "myproject"); + try { + overrides.put("google-compute.credential", toStringAndClose(getClass().getResourceAsStream("/testpk.pem"))); + } catch (IOException e) { + Throwables.propagate(e); + } + return overrides; + } + + @Test(enabled = false) + public void testThrowsAuthorizationException() throws Exception { + + Properties properties = new Properties(); + properties.setProperty("oauth.identity", "MOMMA"); + properties.setProperty("oauth.credential", "MiA"); + + ComputeService client = requestsSendResponses(ImmutableMap.of(), createModule(), + properties); + Template template = client.templateBuilder().build(); + Template toMatch = client.templateBuilder().imageId(template.getImage().getId()).build(); + assertEquals(toMatch.getImage(), template.getImage()); + } + + @Test + public void testTemplateMatch() throws Exception { + ImmutableMap requestResponseMap = ImmutableMap. + builder() + .put(requestForScopes(COMPUTE_READONLY_SCOPE), TOKEN_RESPONSE) + .put(LIST_ZONES_REQ, LIST_ZONES_RESPONSE) + .put(LIST_PROJECT_IMAGES_REQUEST, LIST_PROJECT_IMAGES_RESPONSE) + .put(LIST_GOOGLE_IMAGES_REQUEST, LIST_GOOGLE_IMAGES_RESPONSE) + .put(LIST_MACHINE_TYPES_REQUEST, LIST_MACHINE_TYPES_RESPONSE) + .build(); + + ComputeService client = requestsSendResponses(requestResponseMap); + Template template = client.templateBuilder().build(); + Template toMatch = client.templateBuilder().imageId(template.getImage().getId()).build(); + assertEquals(toMatch.getImage(), template.getImage()); + } + + @Test + public void testNetworksAndFirewallDeletedWhenAllGroupNodesAreTerminated() throws IOException { + + HttpRequest deleteNodeRequest = HttpRequest.builder() + .method("DELETE") + .endpoint("https://www.googleapis" + + ".com/compute/v1beta13/projects/myproject/instances/test-delete-networks") + .addHeader("Accept", "application/json") + .addHeader("Authorization", "Bearer " + TOKEN).build(); + + HttpRequest deleteFirewallRequest = HttpRequest.builder() + .method("DELETE") + .endpoint("https://www.googleapis" + + ".com/compute/v1beta13/projects/myproject/firewalls/jclouds-test-delete") + .addHeader("Accept", "application/json") + .addHeader("Authorization", "Bearer " + TOKEN).build(); + + HttpRequest deleteNetworkReqquest = HttpRequest.builder() + .method("DELETE") + .endpoint("https://www.googleapis" + + ".com/compute/v1beta13/projects/myproject/networks/jclouds-test-delete") + .addHeader("Accept", "application/json") + .addHeader("Authorization", "Bearer " + TOKEN).build(); + + List orderedRequests = ImmutableList.builder() + .add(requestForScopes(COMPUTE_READONLY_SCOPE)) + .add(getInstanceRequestForInstance("test-delete-networks")) + .add(LIST_PROJECT_IMAGES_REQUEST) + .add(LIST_GOOGLE_IMAGES_REQUEST) + .add(LIST_ZONES_REQ) + .add(LIST_MACHINE_TYPES_REQUEST) + .add(requestForScopes(COMPUTE_SCOPE)) + .add(deleteNodeRequest) + .add(GET_OPERATION_REQUEST) + .add(getInstanceRequestForInstance("test-delete-networks")) + .add(LIST_PROJECT_IMAGES_REQUEST) + .add(LIST_GOOGLE_IMAGES_REQUEST) + .add(LIST_ZONES_REQ) + .add(LIST_MACHINE_TYPES_REQUEST) + .add(LIST_INSTANCES_REQUEST) + .add(LIST_PROJECT_IMAGES_REQUEST) + .add(LIST_GOOGLE_IMAGES_REQUEST) + .add(LIST_ZONES_REQ) + .add(LIST_MACHINE_TYPES_REQUEST) + .add(deleteFirewallRequest) + .add(GET_OPERATION_REQUEST) + .add(deleteNetworkReqquest) + .add(GET_OPERATION_REQUEST) + .build(); + + + List orderedResponses = ImmutableList.builder() + .add(TOKEN_RESPONSE) + .add(getInstanceResponseForInstanceAndNetworkAndStatus("test-delete-networks", "test-network", Instance + .Status.RUNNING.name())) + .add(LIST_PROJECT_IMAGES_RESPONSE) + .add(LIST_GOOGLE_IMAGES_RESPONSE) + .add(LIST_ZONES_RESPONSE) + .add(LIST_MACHINE_TYPES_RESPONSE) + .add(TOKEN_RESPONSE) + .add(SUCESSFULL_OPERATION_RESPONSE) + .add(GET_OPERATION_RESPONSE) + .add(getInstanceResponseForInstanceAndNetworkAndStatus("test-delete-networks", "test-network", Instance + .Status.TERMINATED.name())) + .add(LIST_PROJECT_IMAGES_RESPONSE) + .add(LIST_GOOGLE_IMAGES_RESPONSE) + .add(LIST_ZONES_RESPONSE) + .add(LIST_MACHINE_TYPES_RESPONSE) + .add(getListInstancesResponseForSingleInstanceAndNetworkAndStatus("test-delete-networks", + "test-network", Instance + .Status.TERMINATED.name())) + .add(LIST_PROJECT_IMAGES_RESPONSE) + .add(LIST_GOOGLE_IMAGES_RESPONSE) + .add(LIST_ZONES_RESPONSE) + .add(LIST_MACHINE_TYPES_RESPONSE) + .add(SUCESSFULL_OPERATION_RESPONSE) + .add(GET_OPERATION_RESPONSE) + .add(SUCESSFULL_OPERATION_RESPONSE) + .add(GET_OPERATION_RESPONSE) + .build(); + + ComputeService client = orderedRequestsSendResponses(orderedRequests, orderedResponses); + client.destroyNode("test-delete-networks"); + + } + + public void testListLocationsWhenResponseIs2xx() throws Exception { + + ImmutableMap requestResponseMap = ImmutableMap. + builder() + .put(requestForScopes(COMPUTE_READONLY_SCOPE), TOKEN_RESPONSE) + .put(LIST_ZONES_REQ, LIST_ZONES_RESPONSE) + .put(LIST_INSTANCES_REQUEST, LIST_INSTANCES_RESPONSE) + .put(LIST_PROJECT_IMAGES_REQUEST, LIST_PROJECT_IMAGES_RESPONSE) + .put(LIST_GOOGLE_IMAGES_REQUEST, LIST_GOOGLE_IMAGES_RESPONSE) + .put(LIST_MACHINE_TYPES_REQUEST, LIST_MACHINE_TYPES_RESPONSE) + .build(); + + ComputeService apiWhenServersExist = requestsSendResponses(requestResponseMap); + + Set locations = apiWhenServersExist.listAssignableLocations(); + + assertNotNull(locations); + assertEquals(locations.size(), 2); + assertEquals(locations.iterator().next().getId(), "us-central1-a"); + + assertNotNull(apiWhenServersExist.listNodes()); + assertEquals(apiWhenServersExist.listNodes().size(), 1); + assertEquals(apiWhenServersExist.listNodes().iterator().next().getId(), "test-0"); + assertEquals(apiWhenServersExist.listNodes().iterator().next().getName(), "test-0"); + } + + @Test(dependsOnMethods = "testListLocationsWhenResponseIs2xx") + public void testCreateNodeWhenNetworkNorFirewallExistDoesNotExist() throws RunNodesException, IOException { + + + String payload = Strings2.toStringAndClose(InstanceApiExpectTest.class.getResourceAsStream("/instance_get.json")); + payload = payload.replace("test-0", "test-1"); + + HttpResponse getInstanceResponse = HttpResponse.builder().statusCode(200) + .payload(payloadFromStringWithContentType(payload, "application/json")).build(); + + List orderedRequests = ImmutableList.builder() + .add(requestForScopes(COMPUTE_READONLY_SCOPE)) + .add(LIST_ZONES_REQ) + .add(LIST_PROJECT_IMAGES_REQUEST) + .add(LIST_GOOGLE_IMAGES_REQUEST) + .add(LIST_MACHINE_TYPES_REQUEST) + .add(GET_NETWORK_REQUEST) + .add(requestForScopes(COMPUTE_SCOPE)) + .add(INSERT_NETWORK_REQUEST) + .add(GET_OPERATION_REQUEST) + .add(GET_NETWORK_REQUEST) + .add(GET_FIREWALL_REQUEST) + .add(INSERT_FIREWALL_REQUEST) + .add(GET_OPERATION_REQUEST) + .add(LIST_INSTANCES_REQUEST) + .add(LIST_PROJECT_IMAGES_REQUEST) + .add(LIST_GOOGLE_IMAGES_REQUEST) + .add(LIST_ZONES_REQ) + .add(LIST_MACHINE_TYPES_REQUEST) + .add(createInstanceRequestForInstance("test-1", "jclouds-test", openSshKey)) + .add(GET_OPERATION_REQUEST) + .add(getInstanceRequestForInstance("test-1")) + .add(LIST_PROJECT_IMAGES_REQUEST) + .add(LIST_GOOGLE_IMAGES_REQUEST) + .add(LIST_ZONES_REQ) + .add(LIST_MACHINE_TYPES_REQUEST) + .build(); + + List orderedResponses = ImmutableList.builder() + .add(TOKEN_RESPONSE) + .add(LIST_ZONES_RESPONSE) + .add(LIST_PROJECT_IMAGES_RESPONSE) + .add(LIST_GOOGLE_IMAGES_RESPONSE) + .add(LIST_MACHINE_TYPES_RESPONSE) + .add(HttpResponse.builder().statusCode(404).build()) + .add(TOKEN_RESPONSE) + .add(SUCESSFULL_OPERATION_RESPONSE) + .add(GET_OPERATION_RESPONSE) + .add(GET_NETWORK_RESPONSE) + .add(HttpResponse.builder().statusCode(404).build()) + .add(SUCESSFULL_OPERATION_RESPONSE) + .add(GET_OPERATION_RESPONSE) + .add(LIST_INSTANCES_RESPONSE) + .add(LIST_PROJECT_IMAGES_RESPONSE) + .add(LIST_GOOGLE_IMAGES_RESPONSE) + .add(LIST_ZONES_RESPONSE) + .add(LIST_MACHINE_TYPES_RESPONSE) + .add(SUCESSFULL_OPERATION_RESPONSE) + .add(GET_OPERATION_RESPONSE) + .add(getInstanceResponse) + .add(LIST_PROJECT_IMAGES_RESPONSE) + .add(LIST_GOOGLE_IMAGES_RESPONSE) + .add(LIST_ZONES_RESPONSE) + .add(LIST_MACHINE_TYPES_RESPONSE) + .build(); + + + ComputeService computeService = orderedRequestsSendResponses(orderedRequests, orderedResponses); + + GoogleComputeTemplateOptions options = computeService.templateOptions().as(GoogleComputeTemplateOptions.class); + + getOnlyElement(computeService.createNodesInGroup("test", 1, options)); + } +} + diff --git a/labs/google-compute/src/test/java/org/jclouds/googlecompute/compute/GoogleComputeServiceLiveTest.java b/labs/google-compute/src/test/java/org/jclouds/googlecompute/compute/GoogleComputeServiceLiveTest.java new file mode 100644 index 0000000000..d2a05c7f06 --- /dev/null +++ b/labs/google-compute/src/test/java/org/jclouds/googlecompute/compute/GoogleComputeServiceLiveTest.java @@ -0,0 +1,96 @@ +/* + * Licensed to jclouds, Inc. (jclouds) under one or more + * contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. jclouds licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.jclouds.googlecompute.compute; + +import com.google.common.collect.ImmutableMap; +import com.google.inject.Module; +import org.jclouds.compute.domain.NodeMetadata; +import org.jclouds.compute.internal.BaseComputeServiceLiveTest; +import org.jclouds.oauth.v2.OAuthTestUtils; +import org.jclouds.sshj.config.SshjSshClientModule; +import org.testng.annotations.Test; + +import java.util.Properties; + +import static org.testng.Assert.assertTrue; + +/** + * @author David Alves + */ +@Test(groups = "live", singleThreaded = true) +public class GoogleComputeServiceLiveTest extends BaseComputeServiceLiveTest { + + public GoogleComputeServiceLiveTest() { + provider = "google-compute"; + } + + /** + * Nodes may have additional metadata entries (particularly they may have an "sshKeys" entry) + */ + protected void checkUserMetadataInNodeEquals(NodeMetadata node, ImmutableMap userMetadata) { + assertTrue(node.getUserMetadata().keySet().containsAll(userMetadata.keySet())); + } + + // do not run until the auth exception problem is figured out. + @Test(enabled = false) + @Override + public void testCorrectAuthException() throws Exception { + } + + // reboot is not supported by GCE + @Test(enabled = true, dependsOnMethods = "testGet") + public void testReboot() throws Exception { + } + + // suspend/Resume is not supported by GCE + @Test(enabled = true, dependsOnMethods = "testReboot") + public void testSuspendResume() throws Exception { + } + + @Test(enabled = true, dependsOnMethods = "testSuspendResume") + @Override + public void testGetNodesWithDetails() throws Exception { + super.testGetNodesWithDetails(); + } + + @Test(enabled = true, dependsOnMethods = "testSuspendResume") + @Override + public void testListNodes() throws Exception { + super.testListNodes(); + } + + @Test(enabled = true, dependsOnMethods = {"testListNodes", "testGetNodesWithDetails"}) + @Override + public void testDestroyNodes() { + super.testDestroyNodes(); + } + + @Override + protected Module getSshModule() { + return new SshjSshClientModule(); + } + + @Override + protected Properties setupProperties() { + Properties properties = super.setupProperties(); + OAuthTestUtils.setCredentialFromPemFile(properties, "google-compute.credential"); + return properties; + } +} diff --git a/labs/google-compute/src/test/java/org/jclouds/googlecompute/compute/functions/GoogleComputeImageToImageTest.java b/labs/google-compute/src/test/java/org/jclouds/googlecompute/compute/functions/GoogleComputeImageToImageTest.java new file mode 100644 index 0000000000..4803e35298 --- /dev/null +++ b/labs/google-compute/src/test/java/org/jclouds/googlecompute/compute/functions/GoogleComputeImageToImageTest.java @@ -0,0 +1,66 @@ +/* + * Licensed to jclouds, Inc. (jclouds) under one or more + * contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. jclouds licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.jclouds.googlecompute.compute.functions; + +import org.jclouds.compute.domain.OsFamily; +import org.jclouds.googlecompute.domain.Image; +import org.testng.annotations.Test; + +import java.net.URI; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertSame; + +/** + * @author David Alves + */ +@Test(groups = "unit") +public class GoogleComputeImageToImageTest { + + Image.Builder imageBuilder = Image.builder() + .id("1234") + .selfLink(URI.create("http://test.com")) + .sourceType("RAW") + .description("") + .rawDisk(Image.RawDisk.builder().source("").containerType("TAR").build()); + + public void testArbitratyImageName() { + GoogleComputeImageToImage imageToImage = new GoogleComputeImageToImage(); + Image image = imageBuilder.name("arbitratyname").build(); + org.jclouds.compute.domain.Image transformed = imageToImage.apply(image); + assertEquals(transformed.getName(), image.getName()); + assertEquals(transformed.getId(), image.getName()); + assertEquals(transformed.getProviderId(), image.getId()); + assertSame(transformed.getOperatingSystem().getFamily(), OsFamily.LINUX); + } + + public void testWellFormedImageName() { + GoogleComputeImageToImage imageToImage = new GoogleComputeImageToImage(); + Image image = imageBuilder.name("ubuntu-12-04-v123123").build(); + org.jclouds.compute.domain.Image transformed = imageToImage.apply(image); + assertEquals(transformed.getName(), image.getName()); + assertEquals(transformed.getId(), image.getName()); + assertEquals(transformed.getProviderId(), image.getId()); + assertSame(transformed.getOperatingSystem().getFamily(), OsFamily.UBUNTU); + assertEquals(transformed.getOperatingSystem().getVersion(), "12.04"); + } + + +} diff --git a/labs/google-compute/src/test/java/org/jclouds/googlecompute/compute/functions/OrphanedGroupsFromDeadNodesTest.java b/labs/google-compute/src/test/java/org/jclouds/googlecompute/compute/functions/OrphanedGroupsFromDeadNodesTest.java new file mode 100644 index 0000000000..7cfea107e5 --- /dev/null +++ b/labs/google-compute/src/test/java/org/jclouds/googlecompute/compute/functions/OrphanedGroupsFromDeadNodesTest.java @@ -0,0 +1,141 @@ +/* + * Licensed to jclouds, Inc. (jclouds) under one or more + * contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. jclouds licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.jclouds.googlecompute.compute.functions; + +import com.google.common.base.Predicate; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Sets; +import org.easymock.EasyMock; +import org.jclouds.compute.ComputeService; +import org.jclouds.compute.domain.ComputeMetadata; +import org.jclouds.compute.domain.NodeMetadata; +import org.jclouds.compute.domain.internal.NodeMetadataImpl; +import org.jclouds.googlecompute.compute.predicates.AllNodesInGroupTerminated; +import org.testng.annotations.Test; + +import java.util.Set; + +import static org.easymock.EasyMock.createMock; +import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.replay; +import static org.testng.Assert.assertSame; +import static org.testng.Assert.assertTrue; + +/** + * @author David Alves + */ +public class OrphanedGroupsFromDeadNodesTest { + + private static class IdAndGroupOnlyNodeMetadata extends NodeMetadataImpl { + + public IdAndGroupOnlyNodeMetadata(String id, String group, Status status) { + super(null, null, id, null, null, ImmutableMap.of(), ImmutableSet.of(), group, null, + null, null, status, null, 0, ImmutableSet.of(), ImmutableSet.of(), null, null); + } + } + + + @Test + public void testDetectsAllOrphanedGroupsWhenAllNodesTerminated() { + + Set deadNodesGroup1 = (Set) ImmutableSet.builder() + .add(new IdAndGroupOnlyNodeMetadata("a", "1", NodeMetadata.Status.TERMINATED)).build(); + + Set deadNodesGroup2 = (Set) ImmutableSet.builder() + .add(new IdAndGroupOnlyNodeMetadata("b", "2", NodeMetadata.Status.TERMINATED)).build(); + + Set allDeadNodes = Sets.union(deadNodesGroup1, deadNodesGroup2); + + ComputeService mock = createMock(ComputeService.class); + expect(mock.listNodesDetailsMatching(EasyMock.>anyObject())) + .andReturn((Set) deadNodesGroup1).once(); + expect(mock.listNodesDetailsMatching(EasyMock.>anyObject())) + .andReturn((Set) deadNodesGroup2).once(); + + replay(mock); + + OrphanedGroupsFromDeadNodes orphanedGroupsFromDeadNodes = new OrphanedGroupsFromDeadNodes(new + AllNodesInGroupTerminated(mock)); + + Set orphanedGroups = orphanedGroupsFromDeadNodes.apply(allDeadNodes); + + assertSame(orphanedGroups.size(), 2); + assertTrue(orphanedGroups.contains("1")); + assertTrue(orphanedGroups.contains("2")); + } + + @Test + public void testDetectsAllOrphanedGroupsWhenSomeNodesTerminatedAndOtherMissing() { + + Set deadNodesGroup1 = (Set) ImmutableSet.builder() + .add(new IdAndGroupOnlyNodeMetadata("a", "1", NodeMetadata.Status.TERMINATED)).build(); + + Set deadNodesGroup2 = (Set) ImmutableSet.builder() + .add(new IdAndGroupOnlyNodeMetadata("b", "2", NodeMetadata.Status.TERMINATED)).build(); + + Set allDeadNodes = Sets.union(deadNodesGroup1, deadNodesGroup2); + + ComputeService mock = createMock(ComputeService.class); + expect(mock.listNodesDetailsMatching(EasyMock.>anyObject())) + .andReturn((Set) deadNodesGroup1).once(); + expect(mock.listNodesDetailsMatching(EasyMock.>anyObject())) + .andReturn((Set) ImmutableSet.of()).once(); + + replay(mock); + + OrphanedGroupsFromDeadNodes orphanedGroupsFromDeadNodes = new OrphanedGroupsFromDeadNodes(new + AllNodesInGroupTerminated(mock)); + + Set orphanedGroups = orphanedGroupsFromDeadNodes.apply(allDeadNodes); + + assertSame(orphanedGroups.size(), 2); + assertTrue(orphanedGroups.contains("1")); + assertTrue(orphanedGroups.contains("2")); + } + + @Test + public void testDetectsAllOrphanedGroupsWhenSomeNodesAreAlive() { + + Set deadNodesGroup1 = (Set) ImmutableSet.builder() + .add(new IdAndGroupOnlyNodeMetadata("a", "1", NodeMetadata.Status.TERMINATED)).build(); + + Set deadNodesGroup2 = (Set) ImmutableSet.builder() + .add(new IdAndGroupOnlyNodeMetadata("b", "2", NodeMetadata.Status.RUNNING)).build(); + + Set allDeadNodes = Sets.union(deadNodesGroup1, deadNodesGroup2); + + ComputeService mock = createMock(ComputeService.class); + expect(mock.listNodesDetailsMatching(EasyMock.>anyObject())) + .andReturn((Set) deadNodesGroup1).once(); + expect(mock.listNodesDetailsMatching(EasyMock.>anyObject())) + .andReturn((Set) deadNodesGroup2).once(); + + replay(mock); + + OrphanedGroupsFromDeadNodes orphanedGroupsFromDeadNodes = new OrphanedGroupsFromDeadNodes(new + AllNodesInGroupTerminated(mock)); + + Set orphanedGroups = orphanedGroupsFromDeadNodes.apply(allDeadNodes); + + assertSame(orphanedGroups.size(), 1); + assertTrue(orphanedGroups.contains("1")); + } +} diff --git a/labs/google-compute/src/test/java/org/jclouds/googlecompute/features/NetworkApiExpectTest.java b/labs/google-compute/src/test/java/org/jclouds/googlecompute/features/NetworkApiExpectTest.java index b25a7e640c..a3160fe215 100644 --- a/labs/google-compute/src/test/java/org/jclouds/googlecompute/features/NetworkApiExpectTest.java +++ b/labs/google-compute/src/test/java/org/jclouds/googlecompute/features/NetworkApiExpectTest.java @@ -93,7 +93,7 @@ public class NetworkApiExpectTest extends BaseGoogleComputeApiExpectTest { TOKEN_RESPONSE, insert, insertNetworkResponse).getNetworkApiForProject("myproject"); - assertEquals(api.createInIPv4Range("test-network", "10.0.1.0/8"), new ParseOperationTest().expected()); + assertEquals(api.createInIPv4Range("test-network", "10.0.0.0/8"), new ParseOperationTest().expected()); } public void testDeleteNetworkResponseIs2xx() { diff --git a/labs/google-compute/src/test/java/org/jclouds/googlecompute/features/ZoneApiExpectTest.java b/labs/google-compute/src/test/java/org/jclouds/googlecompute/features/ZoneApiExpectTest.java index c73868be91..ddac2a6f38 100644 --- a/labs/google-compute/src/test/java/org/jclouds/googlecompute/features/ZoneApiExpectTest.java +++ b/labs/google-compute/src/test/java/org/jclouds/googlecompute/features/ZoneApiExpectTest.java @@ -37,12 +37,12 @@ import static org.testng.Assert.assertTrue; @Test(groups = "unit") public class ZoneApiExpectTest extends BaseGoogleComputeApiExpectTest { - public static final String ZONES_URL_PREFIX = "https://www.googleapis.com/compute/v1beta13/projects/google/zones"; + public static final String ZONES_URL_PREFIX = "https://www.googleapis.com/compute/v1beta13/projects/myproject/zones"; public static final HttpRequest GET_ZONE_REQ = HttpRequest .builder() .method("GET") - .endpoint(ZONES_URL_PREFIX + "/us-central2-a") + .endpoint(ZONES_URL_PREFIX + "/us-central1-a") .addHeader("Accept", "application/json") .addHeader("Authorization", "Bearer " + TOKEN).build(); @@ -64,9 +64,9 @@ public class ZoneApiExpectTest extends BaseGoogleComputeApiExpectTest { .payload(payloadFromResource("/zone_get.json")).build(); ZoneApi api = requestsSendResponses(requestForScopes(COMPUTE_READONLY_SCOPE), - TOKEN_RESPONSE, GET_ZONE_REQ, operationResponse).getZoneApiForProject("google"); + TOKEN_RESPONSE, GET_ZONE_REQ, operationResponse).getZoneApiForProject("myproject"); - assertEquals(api.get("us-central2-a"), + assertEquals(api.get("us-central1-a"), new ParseZoneTest().expected()); } @@ -75,15 +75,15 @@ public class ZoneApiExpectTest extends BaseGoogleComputeApiExpectTest { HttpResponse operationResponse = HttpResponse.builder().statusCode(404).build(); ZoneApi api = requestsSendResponses(requestForScopes(COMPUTE_READONLY_SCOPE), - TOKEN_RESPONSE, GET_ZONE_REQ, operationResponse).getZoneApiForProject("google"); + TOKEN_RESPONSE, GET_ZONE_REQ, operationResponse).getZoneApiForProject("myproject"); - assertNull(api.get("us-central2-a")); + assertNull(api.get("us-central1-a")); } public void testListZoneNoOptionsResponseIs2xx() throws Exception { ZoneApi api = requestsSendResponses(requestForScopes(COMPUTE_READONLY_SCOPE), - TOKEN_RESPONSE, LIST_ZONES_REQ, LIST_ZONES_RESPONSE).getZoneApiForProject("google"); + TOKEN_RESPONSE, LIST_ZONES_REQ, LIST_ZONES_RESPONSE).getZoneApiForProject("myproject"); assertEquals(api.listFirstPage().toString(), new ParseZoneListTest().expected().toString()); @@ -94,7 +94,7 @@ public class ZoneApiExpectTest extends BaseGoogleComputeApiExpectTest { HttpResponse operationResponse = HttpResponse.builder().statusCode(404).build(); ZoneApi api = requestsSendResponses(requestForScopes(COMPUTE_READONLY_SCOPE), - TOKEN_RESPONSE, LIST_ZONES_REQ, operationResponse).getZoneApiForProject("google"); + TOKEN_RESPONSE, LIST_ZONES_REQ, operationResponse).getZoneApiForProject("myproject"); assertTrue(api.list().concat().isEmpty()); } diff --git a/labs/google-compute/src/test/java/org/jclouds/googlecompute/internal/BaseGoogleComputeExpectTest.java b/labs/google-compute/src/test/java/org/jclouds/googlecompute/internal/BaseGoogleComputeExpectTest.java index e3ec5b558b..68f17b0480 100644 --- a/labs/google-compute/src/test/java/org/jclouds/googlecompute/internal/BaseGoogleComputeExpectTest.java +++ b/labs/google-compute/src/test/java/org/jclouds/googlecompute/internal/BaseGoogleComputeExpectTest.java @@ -20,29 +20,49 @@ package org.jclouds.googlecompute.internal; import com.google.common.base.Joiner; import com.google.common.base.Supplier; -import com.google.common.base.Throwables; import com.google.common.base.Ticker; import com.google.inject.Binder; import com.google.inject.Module; import com.google.inject.TypeLiteral; import org.jclouds.collect.PagedIterable; import org.jclouds.collect.PagedIterables; +import org.jclouds.crypto.Crypto; import org.jclouds.googlecompute.domain.ListPage; import org.jclouds.http.HttpRequest; import org.jclouds.http.HttpResponse; import org.jclouds.io.Payload; import org.jclouds.oauth.v2.OAuthConstants; import org.jclouds.rest.internal.BaseRestApiExpectTest; +import org.jclouds.ssh.SshKeys; import org.jclouds.util.Strings2; import javax.ws.rs.core.MediaType; import java.io.IOException; import java.net.URI; +import java.security.KeyFactory; +import java.security.KeyPair; +import java.security.KeyPairGenerator; +import java.security.NoSuchAlgorithmException; +import java.security.PrivateKey; +import java.security.PublicKey; +import java.security.SecureRandom; +import java.security.interfaces.RSAPublicKey; +import java.security.spec.InvalidKeySpecException; import java.util.Properties; import java.util.concurrent.atomic.AtomicInteger; import static com.google.common.base.Charsets.UTF_8; +import static com.google.common.base.Throwables.propagate; import static com.google.common.io.BaseEncoding.base64Url; +import static org.easymock.EasyMock.createMock; +import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.expectLastCall; +import static org.easymock.EasyMock.replay; +import static org.jclouds.crypto.Pems.privateKeySpec; +import static org.jclouds.crypto.Pems.publicKeySpec; +import static org.jclouds.crypto.PemsTest.PRIVATE_KEY; +import static org.jclouds.crypto.PemsTest.PUBLIC_KEY; +import static org.jclouds.io.Payloads.newStringPayload; /** * @author Adrian Cole @@ -67,6 +87,8 @@ public class BaseGoogleComputeExpectTest extends BaseRestApiExpectTest { " \"expires_in\" : 3600\n" + "}")).build(); + protected String openSshKey; + public BaseGoogleComputeExpectTest() { provider = "google-compute"; @@ -74,16 +96,43 @@ public class BaseGoogleComputeExpectTest extends BaseRestApiExpectTest { @Override protected Module createModule() { + + return new Module() { @Override public void configure(Binder binder) { + // predictable time binder.bind(Ticker.class).toInstance(new Ticker() { @Override public long read() { return 0; } }); - // predictable node names + try { + KeyFactory keyfactory = KeyFactory.getInstance("RSA"); + PrivateKey privateKey = keyfactory.generatePrivate(privateKeySpec(newStringPayload + (PRIVATE_KEY))); + PublicKey publicKey = keyfactory.generatePublic(publicKeySpec(newStringPayload(PUBLIC_KEY))); + KeyPair keyPair = new KeyPair(publicKey, privateKey); + openSshKey = SshKeys.encodeAsOpenSSH(RSAPublicKey.class.cast(publicKey)); + final Crypto crypto = createMock(Crypto.class); + KeyPairGenerator rsaKeyPairGenerator = createMock(KeyPairGenerator.class); + final SecureRandom secureRandom = createMock(SecureRandom.class); + expect(crypto.rsaKeyPairGenerator()).andReturn(rsaKeyPairGenerator).anyTimes(); + rsaKeyPairGenerator.initialize(2048, secureRandom); + expectLastCall().anyTimes(); + expect(rsaKeyPairGenerator.genKeyPair()).andReturn(keyPair).anyTimes(); + replay(crypto, rsaKeyPairGenerator, secureRandom); + binder.bind(Crypto.class).toInstance(crypto); + binder.bind(SecureRandom.class).toInstance(secureRandom); + } catch (NoSuchAlgorithmException e) { + propagate(e); + } catch (InvalidKeySpecException e) { + propagate(e); + } catch (IOException e) { + propagate(e); + } + // predictable node names final AtomicInteger suffix = new AtomicInteger(); binder.bind(new TypeLiteral>() { }).toInstance(new Supplier() { @@ -134,7 +183,7 @@ public class BaseGoogleComputeExpectTest extends BaseRestApiExpectTest { return payloadFromString(Strings2.toStringAndClose(BaseGoogleComputeExpectTest.class.getResourceAsStream (resource))); } catch (IOException e) { - throw Throwables.propagate(e); + throw propagate(e); } } diff --git a/labs/google-compute/src/test/java/org/jclouds/googlecompute/internal/BaseGoogleComputeServiceContextExpectTest.java b/labs/google-compute/src/test/java/org/jclouds/googlecompute/internal/BaseGoogleComputeServiceContextExpectTest.java new file mode 100644 index 0000000000..2474fbdc64 --- /dev/null +++ b/labs/google-compute/src/test/java/org/jclouds/googlecompute/internal/BaseGoogleComputeServiceContextExpectTest.java @@ -0,0 +1,54 @@ +/* + * Licensed to jclouds, Inc. (jclouds) under one or more + * contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. jclouds licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.jclouds.googlecompute.internal; + +import com.google.common.base.Function; +import com.google.inject.Module; +import org.jclouds.apis.ApiMetadata; +import org.jclouds.compute.ComputeServiceContext; +import org.jclouds.googlecompute.GoogleComputeApiMetadata; +import org.jclouds.http.HttpRequest; +import org.jclouds.http.HttpResponse; + +import java.util.Properties; + +/** + * @author David Alves + */ +public abstract class BaseGoogleComputeServiceContextExpectTest extends BaseGoogleComputeExpectTest implements + Function { + + + @Override + public T createClient(Function fn, Module module, Properties props) { + return apply(createComputeServiceContext(fn, module, props)); + } + + private ComputeServiceContext createComputeServiceContext(Function fn, Module module, + Properties props) { + return createInjector(fn, module, props).getInstance(ComputeServiceContext.class); + } + + @Override + protected ApiMetadata createApiMetadata() { + return new GoogleComputeApiMetadata(); + } + +} diff --git a/labs/google-compute/src/test/java/org/jclouds/googlecompute/internal/BaseGoogleComputeServiceExpectTest.java b/labs/google-compute/src/test/java/org/jclouds/googlecompute/internal/BaseGoogleComputeServiceExpectTest.java new file mode 100644 index 0000000000..156879da18 --- /dev/null +++ b/labs/google-compute/src/test/java/org/jclouds/googlecompute/internal/BaseGoogleComputeServiceExpectTest.java @@ -0,0 +1,34 @@ +/* + * Licensed to jclouds, Inc. (jclouds) under one or more + * contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. jclouds licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.jclouds.googlecompute.internal; + +import org.jclouds.compute.ComputeService; +import org.jclouds.compute.ComputeServiceContext; + +/** + * @author David Alves + */ +public class BaseGoogleComputeServiceExpectTest extends BaseGoogleComputeServiceContextExpectTest { + + @Override + public ComputeService apply(ComputeServiceContext input) { + return input.getComputeService(); + } +} diff --git a/labs/google-compute/src/test/java/org/jclouds/googlecompute/parse/ParseNetworkTest.java b/labs/google-compute/src/test/java/org/jclouds/googlecompute/parse/ParseNetworkTest.java index 00940f7f8f..7a7634abd2 100644 --- a/labs/google-compute/src/test/java/org/jclouds/googlecompute/parse/ParseNetworkTest.java +++ b/labs/google-compute/src/test/java/org/jclouds/googlecompute/parse/ParseNetworkTest.java @@ -46,8 +46,8 @@ public class ParseNetworkTest extends BaseGoogleComputeParseTest { .selfLink(URI.create("https://www.googleapis.com/compute/v1beta13/projects/myproject/networks/jclouds-test")) .name("default") .description("Default network for the project") - .IPv4Range("10.240.0.0/16") - .gatewayIPv4("10.240.0.1") + .IPv4Range("10.0.0.0/8") + .gatewayIPv4("10.0.0.1") .build(); } } diff --git a/labs/google-compute/src/test/java/org/jclouds/googlecompute/parse/ParseZoneListTest.java b/labs/google-compute/src/test/java/org/jclouds/googlecompute/parse/ParseZoneListTest.java index 6f270041b8..fe1777882b 100644 --- a/labs/google-compute/src/test/java/org/jclouds/googlecompute/parse/ParseZoneListTest.java +++ b/labs/google-compute/src/test/java/org/jclouds/googlecompute/parse/ParseZoneListTest.java @@ -47,8 +47,8 @@ public class ParseZoneListTest extends BaseGoogleComputeParseTest public ListPage expected() { return ListPage.builder() .kind(Resource.Kind.ZONE_LIST) - .id("projects/google/zones") - .selfLink(URI.create("https://www.googleapis.com/compute/v1beta13/projects/google/zones")) + .id("projects/myproject/zones") + .selfLink(URI.create("https://www.googleapis.com/compute/v1beta13/projects/myproject/zones")) .items(ImmutableSet.of( new ParseZoneTest().expected() , Zone.builder() @@ -56,7 +56,7 @@ public class ParseZoneListTest extends BaseGoogleComputeParseTest .creationTimestamp(new SimpleDateFormatDateService().iso8601DateParse ("2012-10-24T20:13:19.271")) .selfLink(URI.create("https://www.googleapis" + - ".com/compute/v1beta13/projects/google/zones/us-east1-a")) + ".com/compute/v1beta13/projects/myproject/zones/us-east1-a")) .name("us-east1-a") .description("us-east1-a") .status(Zone.Status.UP) diff --git a/labs/google-compute/src/test/java/org/jclouds/googlecompute/parse/ParseZoneTest.java b/labs/google-compute/src/test/java/org/jclouds/googlecompute/parse/ParseZoneTest.java index 2cec9c4d6a..ffdd6643e3 100644 --- a/labs/google-compute/src/test/java/org/jclouds/googlecompute/parse/ParseZoneTest.java +++ b/labs/google-compute/src/test/java/org/jclouds/googlecompute/parse/ParseZoneTest.java @@ -45,9 +45,9 @@ public class ParseZoneTest extends BaseGoogleComputeParseTest { return Zone.builder() .id("13020128040171887099") .creationTimestamp(new SimpleDateFormatDateService().iso8601DateParse("2012-10-19T16:42:54.131")) - .selfLink(URI.create("https://www.googleapis.com/compute/v1beta13/projects/google/zones/us-central2-a")) - .name("us-central2-a") - .description("us-central2-a") + .selfLink(URI.create("https://www.googleapis.com/compute/v1beta13/projects/myproject/zones/us-central1-a")) + .name("us-central1-a") + .description("us-central1-a") .status(Zone.Status.DOWN) .addMaintenanceWindow(Zone.MaintenanceWindow.builder() .name("2012-11-10-planned-outage") diff --git a/labs/google-compute/src/test/resources/network_get.json b/labs/google-compute/src/test/resources/network_get.json index be5790148c..1a6a4b1ab3 100644 --- a/labs/google-compute/src/test/resources/network_get.json +++ b/labs/google-compute/src/test/resources/network_get.json @@ -5,6 +5,6 @@ "selfLink": "https://www.googleapis.com/compute/v1beta13/projects/myproject/networks/jclouds-test", "name": "default", "description": "Default network for the project", - "IPv4Range": "10.240.0.0/16", - "gatewayIPv4": "10.240.0.1" + "IPv4Range": "10.0.0.0/8", + "gatewayIPv4": "10.0.0.1" } \ No newline at end of file diff --git a/labs/google-compute/src/test/resources/network_insert.json b/labs/google-compute/src/test/resources/network_insert.json index 6da7cedc81..55e8331fc5 100644 --- a/labs/google-compute/src/test/resources/network_insert.json +++ b/labs/google-compute/src/test/resources/network_insert.json @@ -1 +1 @@ -{"name":"test-network","IPv4Range":"10.0.1.0/8"} \ No newline at end of file +{"name":"test-network","IPv4Range":"10.0.0.0/8"} \ No newline at end of file diff --git a/labs/google-compute/src/test/resources/network_list.json b/labs/google-compute/src/test/resources/network_list.json index 0ce7a7cce5..99ccc49b79 100644 --- a/labs/google-compute/src/test/resources/network_list.json +++ b/labs/google-compute/src/test/resources/network_list.json @@ -11,8 +11,8 @@ "selfLink": "https://www.googleapis.com/compute/v1beta13/projects/myproject/networks/jclouds-test", "name": "default", "description": "Default network for the project", - "IPv4Range": "10.240.0.0/16", - "gatewayIPv4": "10.240.0.1" + "IPv4Range": "10.0.0.0/8", + "gatewayIPv4": "10.0.0.1" } ] } \ No newline at end of file diff --git a/labs/google-compute/src/test/resources/zone_get.json b/labs/google-compute/src/test/resources/zone_get.json index 919a5bd7df..cf3dc09152 100644 --- a/labs/google-compute/src/test/resources/zone_get.json +++ b/labs/google-compute/src/test/resources/zone_get.json @@ -2,9 +2,9 @@ "kind": "compute#zone", "id": "13020128040171887099", "creationTimestamp": "2012-10-19T16:42:54.131", - "selfLink": "https://www.googleapis.com/compute/v1beta13/projects/google/zones/us-central2-a", - "name": "us-central2-a", - "description": "us-central2-a", + "selfLink": "https://www.googleapis.com/compute/v1beta13/projects/myproject/zones/us-central1-a", + "name": "us-central1-a", + "description": "us-central1-a", "status": "DOWN", "maintenanceWindows": [ { diff --git a/labs/google-compute/src/test/resources/zone_list.json b/labs/google-compute/src/test/resources/zone_list.json index ae67048426..8d11f0d9b8 100644 --- a/labs/google-compute/src/test/resources/zone_list.json +++ b/labs/google-compute/src/test/resources/zone_list.json @@ -1,15 +1,15 @@ { "kind": "compute#zoneList", - "id": "projects/google/zones", - "selfLink": "https://www.googleapis.com/compute/v1beta13/projects/google/zones", + "id": "projects/myproject/zones", + "selfLink": "https://www.googleapis.com/compute/v1beta13/projects/myproject/zones", "items": [ { "kind": "compute#zone", "id": "13020128040171887099", "creationTimestamp": "2012-10-19T16:42:54.131", - "selfLink": "https://www.googleapis.com/compute/v1beta13/projects/google/zones/us-central2-a", - "name": "us-central2-a", - "description": "us-central2-a", + "selfLink": "https://www.googleapis.com/compute/v1beta13/projects/myproject/zones/us-central1-a", + "name": "us-central1-a", + "description": "us-central1-a", "status": "DOWN", "maintenanceWindows": [ { @@ -24,7 +24,7 @@ "kind": "compute#zone", "id": "13024414164050619686", "creationTimestamp": "2012-10-24T20:13:19.271", - "selfLink": "https://www.googleapis.com/compute/v1beta13/projects/google/zones/us-east1-a", + "selfLink": "https://www.googleapis.com/compute/v1beta13/projects/myproject/zones/us-east1-a", "name": "us-east1-a", "description": "us-east1-a", "status": "UP",