add support for whitelisting locations

- change location scope to ZONE vs REGION
- edit the README
- fix Region.byName
- add more Regions in Region class
- address initial comments from @nacx
- revert ZONES to REGIONS
- add parser module
- remove parser from HttpApiModule
- use one region in AzureLiveTestUtils
This commit is contained in:
Andrea Turli 2016-08-11 10:52:27 +02:00
parent 1e484fc2ce
commit 204d54d57c
10 changed files with 162 additions and 184 deletions

View File

@ -4,8 +4,6 @@ jclouds Labs - Azure Compute ARM Provider
Build status for azurecomputearm module:
[![Build Status](https://jclouds.ci.cloudbees.com/buildStatus/icon?job=jclouds-labs/org.apache.jclouds.labs$azurecompute-arm)](https://jclouds.ci.cloudbees.com/job/jclouds-labs/org.apache.jclouds.labs$azurecompute-arm/)
## Setting Up Test Credentials
### Create a Service Principal
@ -75,3 +73,24 @@ mvn clean verify -Plive \
-Dtest.oauth.endpoint=https://login.microsoftonline.com/<Tenant-id>/oauth2/token
```
## How to use it
Azure Compute ARM provider works exactly as any other jclouds provider.
Notice that as Azure supports dozens of locations, operations like listImages can be really time-consuming.
To limit the scope of such operations there are some additional properties you may want to use:
```bash
jclouds.azurecompute.arm.publishers
```
which is by default `Canonical,RedHat`
and
```bash
jclouds.regions
```
which is by default `null`. If you want to target only the `north europe` region, you can use
```bash
jclouds.regions="northeurope"
```

View File

@ -16,21 +16,22 @@
*/
package org.jclouds.azurecompute.arm;
import static org.jclouds.reflect.Reflection2.typeToken;
import java.net.URI;
import java.util.Properties;
import org.jclouds.apis.ApiMetadata;
import org.jclouds.azurecompute.arm.config.AzureComputeHttpApiModule;
import org.jclouds.compute.ComputeServiceContext;
import org.jclouds.rest.internal.BaseHttpApiMetadata;
import org.jclouds.oauth.v2.config.OAuthModule;
import org.jclouds.azurecompute.arm.compute.config.AzureComputeServiceContextModule;
import org.jclouds.azurecompute.arm.config.AzureComputeHttpApiModule;
import org.jclouds.azurecompute.arm.config.AzureComputeParserModule;
import org.jclouds.compute.ComputeServiceContext;
import org.jclouds.http.okhttp.config.OkHttpCommandExecutorServiceModule;
import org.jclouds.oauth.v2.config.OAuthModule;
import org.jclouds.rest.internal.BaseHttpApiMetadata;
import com.google.common.collect.ImmutableSet;
import com.google.inject.Module;
import org.jclouds.http.okhttp.config.OkHttpCommandExecutorServiceModule;
import static org.jclouds.reflect.Reflection2.typeToken;
/**
* Implementation of {@link ApiMetadata} for Microsoft Azure Resource Manager REST API
@ -72,6 +73,7 @@ public class AzureManagementApiMetadata extends BaseHttpApiMetadata<AzureCompute
.add(AzureComputeServiceContextModule.class)
.add(OAuthModule.class)
.add(OkHttpCommandExecutorServiceModule.class)
.add(AzureComputeParserModule.class)
.add(AzureComputeHttpApiModule.class).build());
}

View File

@ -16,14 +16,9 @@
*/
package org.jclouds.azurecompute.arm.compute;
import static java.lang.String.format;
import static java.util.concurrent.TimeUnit.SECONDS;
import static org.jclouds.util.Predicates2.retry;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Set;
@ -33,9 +28,6 @@ import javax.inject.Inject;
import javax.inject.Named;
import javax.inject.Singleton;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.Multimap;
import com.google.common.net.UrlEscapers;
import org.jclouds.azurecompute.arm.AzureComputeApi;
import org.jclouds.azurecompute.arm.compute.config.AzureComputeServiceContextModule.AzureComputeConstants;
import org.jclouds.azurecompute.arm.compute.functions.VMImageToImage;
@ -52,27 +44,38 @@ import org.jclouds.azurecompute.arm.domain.Offer;
import org.jclouds.azurecompute.arm.domain.PublicIPAddress;
import org.jclouds.azurecompute.arm.domain.SKU;
import org.jclouds.azurecompute.arm.domain.VMDeployment;
import org.jclouds.azurecompute.arm.domain.VMSize;
import org.jclouds.azurecompute.arm.domain.Version;
import org.jclouds.azurecompute.arm.domain.VirtualMachine;
import org.jclouds.azurecompute.arm.domain.VirtualMachineInstance;
import org.jclouds.azurecompute.arm.features.DeploymentApi;
import org.jclouds.azurecompute.arm.features.OSImageApi;
import org.jclouds.azurecompute.arm.util.BlobHelper;
import org.jclouds.azurecompute.arm.functions.CleanupResources;
import org.jclouds.azurecompute.arm.util.DeploymentTemplateBuilder;
import org.jclouds.compute.ComputeServiceAdapter;
import org.jclouds.compute.domain.Template;
import org.jclouds.compute.reference.ComputeServiceConstants;
import org.jclouds.domain.LoginCredentials;
import org.jclouds.location.reference.LocationConstants;
import org.jclouds.logging.Logger;
import org.jclouds.azurecompute.arm.functions.CleanupResources;
import org.jclouds.azurecompute.arm.domain.VMSize;
import org.jclouds.azurecompute.arm.domain.Version;
import org.jclouds.providers.ProviderMetadata;
import com.google.common.base.Function;
import com.google.common.base.Predicate;
import com.google.common.base.Splitter;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.FluentIterable;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.collect.Multimap;
import com.google.common.collect.Sets;
import com.google.common.base.Splitter;
import com.google.common.net.UrlEscapers;
import static com.google.common.base.Preconditions.checkState;
import static java.lang.String.format;
import static java.util.concurrent.TimeUnit.SECONDS;
import static org.jclouds.util.Predicates2.retry;
/**
* Defines the connection between the {@link AzureComputeApi} implementation and the jclouds
@ -92,9 +95,11 @@ public class AzureComputeServiceAdapter implements ComputeServiceAdapter<VMDeplo
private final AzureComputeConstants azureComputeConstants;
private final ProviderMetadata providerMetadata;
@Inject
AzureComputeServiceAdapter(final AzureComputeApi api, final AzureComputeConstants azureComputeConstants,
CleanupResources cleanupResources) {
CleanupResources cleanupResources, ProviderMetadata providerMetadata) {
this.api = api;
this.azureComputeConstants = azureComputeConstants;
@ -103,6 +108,7 @@ public class AzureComputeServiceAdapter implements ComputeServiceAdapter<VMDeplo
logger.debug("AzureComputeServiceAdapter set azuregroup to: " + azureGroup);
this.cleanupResources = cleanupResources;
this.providerMetadata = providerMetadata;
}
@Override
@ -207,9 +213,8 @@ public class AzureComputeServiceAdapter implements ComputeServiceAdapter<VMDeplo
// }
}
private void getImagesFromPublisher(String publisherName, List<VMImage> osImagesRef, String location) {
private List<VMImage> getImagesFromPublisher(String publisherName, String location) {
List<VMImage> osImagesRef = Lists.newArrayList();
OSImageApi osImageApi = api.getOSImageApi(location);
Iterable<Offer> offerList = osImageApi.listOffers(publisherName);
@ -224,14 +229,14 @@ public class AzureComputeServiceAdapter implements ComputeServiceAdapter<VMDeplo
}
}
}
return osImagesRef;
}
private List<VMImage> listImagesByLocation(String location) {
final List<VMImage> osImages = Lists.newArrayList();
Iterable<String> publishers = Splitter.on(',').trimResults().omitEmptyStrings().split(this.azureComputeConstants.azureImagePublishers());
for (String publisher : publishers) {
getImagesFromPublisher(publisher, osImages, location);
osImages.addAll(getImagesFromPublisher(publisher, location));
}
return osImages;
}
@ -240,14 +245,10 @@ public class AzureComputeServiceAdapter implements ComputeServiceAdapter<VMDeplo
public Iterable<VMImage> listImages() {
final List<VMImage> osImages = Lists.newArrayList();
final List<String> locationIds = Lists.newArrayList();
for (Location location : listLocations()){
locationIds.add(location.name());
osImages.addAll(listImagesByLocation(location.name()));
}
checkAndSetImageAvailability(osImages, Sets.newHashSet(locationIds));
// list custom images
List<StorageService> storages = api.getStorageAccountApi(azureGroup).list();
for (StorageService storage : storages) {
@ -257,22 +258,9 @@ public class AzureComputeServiceAdapter implements ComputeServiceAdapter<VMDeplo
"custom", storage.location());
osImages.addAll(images);
}
return osImages;
}
private void checkAndSetImageAvailability(List<VMImage> images, Collection<String> locations) {
Multimap<String, String> map = ArrayListMultimap.create();
for (VMImage image : images) {
map.put( image.offer() + "/" + image.sku(), image.location());
}
///TODO
// for (VMImage image : images) {
// image.globallyAvailable() = map.get(image.offer() + "/" + image.sku()).containsAll(locations);
// }
}
@Override
public VMImage getImage(final String id) {
VMImage image = VMImageToImage.decodeFieldsFromUniqueId(id);
@ -300,28 +288,43 @@ public class AzureComputeServiceAdapter implements ComputeServiceAdapter<VMDeplo
@Override
public Iterable<Location> listLocations() {
final Iterable<String> whiteListZoneName = findWhiteListOfRegions();
List<Location> locations = api.getLocationApi().list();
final Iterable<String> vmLocations = FluentIterable.from(api.getResourceProviderApi().get("Microsoft.Compute"))
.filter(new Predicate<ResourceProviderMetaData>() {
@Override
public boolean apply(ResourceProviderMetaData input) {
return input.resourceType().equals("virtualMachines");
}
})
.transformAndConcat(new Function<ResourceProviderMetaData, Iterable<String>>() {
@Override
public Iterable<String> apply(ResourceProviderMetaData resourceProviderMetaData) {
return resourceProviderMetaData.locations();
}
});
List<ResourceProviderMetaData> resources = api.getResourceProviderApi().get("Microsoft.Compute");
List<Location> locations = FluentIterable.from(api.getLocationApi().list())
.filter(new Predicate<Location>() {
@Override
public boolean apply(Location location) {
return Iterables.contains(vmLocations, location.displayName());
}
})
.filter(new Predicate<Location>() {
@Override
public boolean apply(Location location) {
return whiteListZoneName == null ? true : Iterables.contains(whiteListZoneName, location.name());
}
})
.toList();
final List<String> vmLocations = new ArrayList<String>();
return locations;
}
for (ResourceProviderMetaData m : resources){
if (m.resourceType().equals("virtualMachines")){
vmLocations.addAll(m.locations());
break;
}
}
Iterable<Location> result = Iterables.filter(locations, new Predicate<Location>() {
@Override
public boolean apply(Location input) {
return vmLocations.contains(input.displayName());
}
});
return result;
private Iterable<String> findWhiteListOfRegions() {
if (providerMetadata.getDefaultProperties().get(LocationConstants.PROPERTY_REGIONS) == null) return null;
return Splitter.on(",").trimResults().split((CharSequence) providerMetadata.getDefaultProperties().get(LocationConstants.PROPERTY_REGIONS));
}
private String getResourceGroupFromId(String id) {

View File

@ -16,8 +16,6 @@
*/
package org.jclouds.azurecompute.arm.compute.functions;
import static com.google.common.collect.Iterables.getOnlyElement;
import javax.inject.Inject;
import javax.inject.Singleton;
@ -30,6 +28,8 @@ import org.jclouds.location.suppliers.all.JustProvider;
import com.google.common.base.Function;
import com.google.common.collect.ImmutableSet;
import static com.google.common.collect.Iterables.getOnlyElement;
/**
* Converts an Location into a Location.
*/
@ -47,20 +47,14 @@ public class LocationToLocation implements Function<Location, org.jclouds.domain
@Override
public org.jclouds.domain.Location apply(final Location location) {
final LocationBuilder builder = new LocationBuilder();
String id = location.id();
int index = id.lastIndexOf('/');
if (index > 0 && (index + 1) < id.length())
id = id.substring(index + 1);
builder.id(id);
builder.id(location.name());
builder.description(location.displayName());
builder.parent(getOnlyElement(justProvider.get()));
builder.scope(LocationScope.REGION);
final Region region = Region.byName(location.name());
final Region region = Region.byName(location.displayName());
if (region != null) {
builder.iso3166Codes(ImmutableSet.of(region.iso3166Code()));
}
return builder.build();
}

View File

@ -15,23 +15,22 @@
* limitations under the License.
*/
package org.jclouds.azurecompute.arm.config;
import org.jclouds.azurecompute.arm.AzureComputeApi;
import org.jclouds.azurecompute.arm.handlers.AzureComputeErrorHandler;
import org.jclouds.azurecompute.arm.util.DeploymentTemplateBuilder;
import org.jclouds.http.HttpErrorHandler;
import org.jclouds.http.annotation.ClientError;
import org.jclouds.http.annotation.Redirection;
import org.jclouds.http.annotation.ServerError;
import org.jclouds.location.suppliers.ImplicitLocationSupplier;
import org.jclouds.location.suppliers.implicit.OnlyLocationOrFirstRegionOptionallyMatchingRegionId;
import org.jclouds.location.suppliers.implicit.FirstRegion;
import org.jclouds.oauth.v2.config.OAuthScopes;
import org.jclouds.rest.ConfiguresHttpApi;
import org.jclouds.rest.config.HttpApiModule;
import org.jclouds.oauth.v2.config.OAuthScopes;
import com.google.inject.assistedinject.FactoryModuleBuilder;
import com.google.inject.Scopes;
import com.google.inject.assistedinject.FactoryModuleBuilder;
@ConfiguresHttpApi
public class AzureComputeHttpApiModule extends HttpApiModule<AzureComputeApi> {
@ -46,14 +45,12 @@ public class AzureComputeHttpApiModule extends HttpApiModule<AzureComputeApi> {
@Override
protected void installLocations() {
super.installLocations();
bind(ImplicitLocationSupplier.class).
to(OnlyLocationOrFirstRegionOptionallyMatchingRegionId.class).
in(Scopes.SINGLETON);
bind(ImplicitLocationSupplier.class).to(FirstRegion.class).in(Scopes.SINGLETON);
}
@Override
protected void configure() {
install(new AzureComputeParserModule());
install(new FactoryModuleBuilder().build(DeploymentTemplateBuilder.Factory.class));
super.configure();
bind(OAuthScopes.class).toInstance(OAuthScopes.NoScopes.create());

View File

@ -16,12 +16,13 @@
*/
package org.jclouds.azurecompute.arm.domain;
import java.util.Arrays;
import java.util.Set;
import com.google.common.base.Function;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
import java.util.Arrays;
import java.util.Set;
/**
* Regions used in Azure.
@ -46,7 +47,14 @@ public enum Region {
JAPAN_WEST("Japan West", "JP-27"),
BRAZIL_SOUTH("Brazil South", "BR"),
AUSTRALIA_EAST("Australia East", "AU-NSW"),
AUSTRALIA_SOUTH_EAST("Australia Southeast", "AU-VIC");
AUSTRALIA_SOUTH_EAST("Australia Southeast", "AU-VIC"),
INDIA_CENTRAL("Central India", "IN-GA"),
INDIA_SOUTH("South India", "IN-TN"),
INDIA_WEST("West India", "IN-MH"),
CHINA_EAST("China East", "CN-SH"),
CHINA_NORTH("China North", "CN-BJ"),
CANADA_CENTRAL("Canada Central", "CA-ON"),
CANADA_EAST("Canada East", "CA-QC");
private final String name;
@ -73,7 +81,6 @@ public enum Region {
return region;
}
}
return null;
}
@ -86,4 +93,5 @@ public enum Region {
}
}));
}
}

View File

@ -16,6 +16,13 @@
*/
package org.jclouds.azurecompute.arm.compute;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import org.jclouds.azurecompute.arm.AzureComputeProviderMetadata;
import org.jclouds.azurecompute.arm.internal.AzureLiveTestUtils;
import org.jclouds.compute.RunScriptOnNodesException;
import org.jclouds.compute.domain.ExecResponse;
import org.jclouds.compute.domain.NodeMetadata;
@ -24,32 +31,25 @@ import org.jclouds.compute.domain.Template;
import org.jclouds.compute.internal.BaseComputeServiceLiveTest;
import org.jclouds.compute.predicates.NodePredicates;
import org.jclouds.domain.LoginCredentials;
import org.jclouds.logging.config.LoggingModule;
import org.jclouds.logging.slf4j.config.SLF4JLoggingModule;
import org.jclouds.providers.ProviderMetadata;
import org.jclouds.scriptbuilder.domain.Statement;
import org.jclouds.scriptbuilder.domain.Statements;
import org.jclouds.scriptbuilder.statements.java.InstallJDK;
import org.jclouds.scriptbuilder.statements.login.AdminAccess;
import org.jclouds.sshj.config.SshjSshClientModule;
import org.testng.annotations.Test;
import org.jclouds.providers.ProviderMetadata;
import org.jclouds.azurecompute.arm.AzureComputeProviderMetadata;
import static org.jclouds.azurecompute.arm.config.AzureComputeProperties.RESOURCE_GROUP_NAME;
import static org.jclouds.compute.config.ComputeServiceProperties.TIMEOUT_NODE_RUNNING;
import static org.jclouds.compute.config.ComputeServiceProperties.TIMEOUT_NODE_SUSPENDED;
import static org.jclouds.compute.config.ComputeServiceProperties.TIMEOUT_NODE_TERMINATED;
import static org.jclouds.compute.config.ComputeServiceProperties.TIMEOUT_PORT_OPEN;
import static org.jclouds.compute.config.ComputeServiceProperties.TIMEOUT_SCRIPT_COMPLETE;
import org.jclouds.azurecompute.arm.internal.AzureLiveTestUtils;
import com.google.inject.Module;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import static com.google.common.base.Preconditions.checkNotNull;
import org.jclouds.logging.slf4j.config.SLF4JLoggingModule;
import org.jclouds.logging.config.LoggingModule;
import static org.jclouds.azurecompute.arm.config.AzureComputeProperties.RESOURCE_GROUP_NAME;
import static org.jclouds.compute.config.ComputeServiceProperties.TIMEOUT_SCRIPT_COMPLETE;
import static org.jclouds.compute.config.ComputeServiceProperties.TIMEOUT_NODE_RUNNING;
import static org.jclouds.compute.config.ComputeServiceProperties.TIMEOUT_PORT_OPEN;
import static org.jclouds.compute.config.ComputeServiceProperties.TIMEOUT_NODE_TERMINATED;
import static org.jclouds.compute.config.ComputeServiceProperties.TIMEOUT_NODE_SUSPENDED;
/**
* Live tests for the {@link org.jclouds.compute.ComputeService} integration.

View File

@ -17,8 +17,10 @@
package org.jclouds.azurecompute.arm.internal;
import java.util.Properties;
import static org.jclouds.oauth.v2.config.OAuthProperties.CREDENTIAL_TYPE;
import static org.jclouds.location.reference.LocationConstants.PROPERTY_REGIONS;
import static org.jclouds.oauth.v2.config.CredentialType.CLIENT_CREDENTIALS_SECRET;
import static org.jclouds.oauth.v2.config.OAuthProperties.CREDENTIAL_TYPE;
public class AzureLiveTestUtils {
@ -28,6 +30,7 @@ public class AzureLiveTestUtils {
properties.put("oauth.credential", "password");
properties.put("oauth.endpoint", "https://login.microsoftonline.com/oauth2/token");
properties.put(CREDENTIAL_TYPE, CLIENT_CREDENTIALS_SECRET.toString());
properties.put(PROPERTY_REGIONS, "northeurope");
return properties;
}
}

View File

@ -0,0 +1,34 @@
<?xml version="1.0"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<configuration>
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>- %msg%n</pattern>
</encoder>
</appender>
<root level="info">
<appender-ref ref="STDOUT"/>
</root>
<logger name="jclouds.compute" level="debug"/>
<logger name="net.schmizz" level="warn"/>
<logger name="jclouds.wire" level="debug"/>
<logger name="jclouds.headers" level="debug"/>
<logger name="jclouds.ssh" level="debug"/>
</configuration>

View File

@ -1,82 +0,0 @@
<?xml version="1.0"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<configuration scan="false">
<appender name="FILE" class="ch.qos.logback.core.FileAppender">
<file>target/test-data/jclouds.log</file>
<encoder>
<Pattern>%d %-5p [%c] [%thread] %m%n</Pattern>
</encoder>
</appender>
<appender name="WIREFILE" class="ch.qos.logback.core.FileAppender">
<file>target/test-data/jclouds-wire.log</file>
<encoder>
<Pattern>%d %-5p [%c] [%thread] %m%n</Pattern>
</encoder>
</appender>
<appender name="COMPUTEFILE" class="ch.qos.logback.core.FileAppender">
<file>target/test-data/jclouds-compute.log</file>
<encoder>
<Pattern>%d %-5p [%c] [%thread] %m%n</Pattern>
</encoder>
</appender>
<appender name="SSHFILE" class="ch.qos.logback.core.FileAppender">
<file>target/test-data/jclouds-ssh.log</file>
<encoder>
<Pattern>%d %-5p [%c] [%thread] %m%n</Pattern>
</encoder>
</appender>
<root>
<level value="warn" />
</root>
<logger name="org.jclouds">
<level value="TRACE" />
<appender-ref ref="FILE" />
</logger>
<logger name="jclouds.wire">
<level value="DEBUG" />
<appender-ref ref="WIREFILE" />
</logger>
<logger name="jclouds.headers">
<level value="DEBUG" />
<appender-ref ref="WIREFILE" />
</logger>
<logger name="jclouds.compute">
<level value="TRACE" />
<appender-ref ref="COMPUTEFILE" />
</logger>
<logger name="jclouds.ssh">
<level value="TRACE" />
<appender-ref ref="SSHFILE" />
</logger>
</configuration>