JCLOUDS-664 Azurecompute-arm ImageExtension

This commit is contained in:
Janne Koskinen 2016-07-07 13:50:59 +03:00 committed by Ignasi Barrera
parent 4f62f40036
commit 3c790ae3ae
9 changed files with 300 additions and 75 deletions

View File

@ -113,6 +113,11 @@
<version>${project.parent.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.jclouds.provider</groupId>
<artifactId>azureblob</artifactId>
<version>${project.parent.version}</version>
</dependency>
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
@ -135,6 +140,11 @@
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.jclouds</groupId>
<artifactId>jclouds-blobstore</artifactId>
<version>${project.parent.version}</version>
</dependency>
</dependencies>
<profiles>

View File

@ -18,7 +18,6 @@ package org.jclouds.azurecompute.arm.compute;
import static java.lang.String.format;
import static java.util.concurrent.TimeUnit.SECONDS;
import static org.jclouds.azurecompute.arm.compute.extensions.AzureComputeImageExtension.CUSTOM_IMAGE_PREFIX;
import static org.jclouds.util.Predicates2.retry;
import java.util.ArrayList;
@ -45,6 +44,7 @@ import org.jclouds.azurecompute.arm.domain.DeploymentBody;
import org.jclouds.azurecompute.arm.domain.DeploymentProperties;
import org.jclouds.azurecompute.arm.domain.NetworkInterfaceCard;
import org.jclouds.azurecompute.arm.domain.ResourceProviderMetaData;
import org.jclouds.azurecompute.arm.domain.StorageService;
import org.jclouds.azurecompute.arm.domain.VMImage;
import org.jclouds.azurecompute.arm.domain.VMHardware;
import org.jclouds.azurecompute.arm.domain.Location;
@ -56,6 +56,7 @@ import org.jclouds.azurecompute.arm.domain.VirtualMachine;
import org.jclouds.azurecompute.arm.domain.VirtualMachineInstance;
import org.jclouds.azurecompute.arm.features.DeploymentApi;
import org.jclouds.azurecompute.arm.features.OSImageApi;
import org.jclouds.azurecompute.arm.util.BlobHelper;
import org.jclouds.azurecompute.arm.util.DeploymentTemplateBuilder;
import org.jclouds.compute.ComputeServiceAdapter;
import org.jclouds.compute.domain.Template;
@ -97,7 +98,6 @@ public class AzureComputeServiceAdapter implements ComputeServiceAdapter<VMDeplo
this.api = api;
this.azureComputeConstants = azureComputeConstants;
this.azureGroup = azureComputeConstants.azureResourceGroup();
logger.debug("AzureComputeServiceAdapter set azuregroup to: " + azureGroup);
@ -219,7 +219,7 @@ public class AzureComputeServiceAdapter implements ComputeServiceAdapter<VMDeplo
for (SKU sku : skuList) {
Iterable<Version> versionList = osImageApi.listVersions(publisherName, offer.name(), sku.name());
for (Version version : versionList) {
VMImage vmImage = VMImage.create(publisherName, offer.name(), sku.name(), version.name(), location, false);
VMImage vmImage = VMImage.create(publisherName, offer.name(), sku.name(), version.name(), location);
osImagesRef.add(vmImage);
}
}
@ -247,6 +247,17 @@ public class AzureComputeServiceAdapter implements ComputeServiceAdapter<VMDeplo
osImages.addAll(listImagesByLocation(location.name()));
}
checkAndSetImageAvailability(osImages, Sets.newHashSet(locationIds));
// list custom images
List<StorageService> storages = api.getStorageAccountApi(azureGroup).list();
for (StorageService storage : storages) {
String name = storage.name();
String key = api.getStorageAccountApi(azureGroup).getKeys(name).key1();
List<VMImage> images = BlobHelper.getImages("jclouds", azureGroup, storage.name(), key,
"custom", storage.location());
osImages.addAll(images);
}
return osImages;
}
@ -264,24 +275,25 @@ public class AzureComputeServiceAdapter implements ComputeServiceAdapter<VMDeplo
@Override
public VMImage getImage(final String id) {
String[] fields = VMImageToImage.decodeFieldsFromUniqueId(id);
if (fields[2].startsWith(CUSTOM_IMAGE_PREFIX)) {
String name = fields[2].substring(CUSTOM_IMAGE_PREFIX.length());
String sku = fields[3];
String version = "1";
VMImage ref = VMImage.create(CUSTOM_IMAGE_PREFIX + azureGroup, CUSTOM_IMAGE_PREFIX + name, sku, version, fields[0], false);
return ref;
VMImage image = VMImageToImage.decodeFieldsFromUniqueId(id);
if (image.custom()) {
String key = api.getStorageAccountApi(azureGroup).getKeys(image.storage()).key1();
if (BlobHelper.customImageExists(image.storage(), key))
return image;
else
return null;
}
String location = fields[0];
String publisher = fields[1];
String offer = fields[2];
String sku = fields[3];
String location = image.location();
String publisher = image.publisher();
String offer = image.offer();
String sku = image.sku();
OSImageApi osImageApi = api.getOSImageApi(location);
List<Version> versions = osImageApi.listVersions(publisher, offer, sku);
if (!versions.isEmpty()) {
return VMImage.create(publisher, offer, sku, versions.get(0).name(), location, false);
return VMImage.create(publisher, offer, sku, versions.get(0).name(), location);
}
return null;
}
@ -415,8 +427,18 @@ public class AzureComputeServiceAdapter implements ComputeServiceAdapter<VMDeplo
List<VMDeployment> vmDeployments = new ArrayList<VMDeployment>();
for (Deployment d : deployments){
// Check that this vm is not generalized and made to custom image
try {
String storageAccountName = d.name().replaceAll("[^A-Za-z0-9 ]", "") + "stor";
String key = api.getStorageAccountApi(azureGroup).getKeys(storageAccountName).key1();
if (!BlobHelper.customImageExists(storageAccountName, key))
vmDeployments.add(convertDeploymentToVMDeployment(d));
}
catch (Exception e) {
// This might happen if there is no custom images but vm is generalized. No need to list
}
}
return vmDeployments;
}

View File

@ -17,18 +17,24 @@
package org.jclouds.azurecompute.arm.compute.extensions;
import com.google.common.base.Predicate;
import com.google.common.base.Supplier;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.common.util.concurrent.UncheckedTimeoutException;
import com.google.gson.internal.LinkedTreeMap;
import com.google.inject.Inject;
import com.google.inject.name.Named;
import org.jclouds.Constants;
import org.jclouds.View;
import org.jclouds.azurecompute.arm.AzureComputeApi;
import org.jclouds.azurecompute.arm.compute.config.AzureComputeServiceContextModule;
import org.jclouds.azurecompute.arm.compute.functions.VMImageToImage;
import org.jclouds.azurecompute.arm.domain.ResourceDefinition;
import org.jclouds.azurecompute.arm.domain.StorageServiceKeys;
import org.jclouds.azurecompute.arm.domain.VMImage;
import org.jclouds.azurecompute.arm.domain.VirtualMachine;
import static java.lang.String.format;
import org.jclouds.azurecompute.arm.util.BlobHelper;
import org.jclouds.compute.domain.CloneImageTemplate;
import org.jclouds.compute.domain.Image;
import org.jclouds.compute.domain.ImageTemplate;
@ -36,27 +42,25 @@ import org.jclouds.compute.domain.ImageTemplateBuilder;
import org.jclouds.compute.extensions.ImageExtension;
import org.jclouds.azurecompute.arm.compute.config.AzureComputeServiceContextModule.AzureComputeConstants;
import static java.lang.String.format;
import static org.jclouds.compute.config.ComputeServiceProperties.TIMEOUT_IMAGE_AVAILABLE;
import static org.jclouds.compute.config.ComputeServiceProperties.TIMEOUT_NODE_SUSPENDED;
import com.google.common.util.concurrent.UncheckedTimeoutException;
import java.net.URI;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Callable;
import static org.jclouds.compute.config.ComputeServiceProperties.TIMEOUT_IMAGE_AVAILABLE;
import static org.jclouds.compute.config.ComputeServiceProperties.TIMEOUT_NODE_SUSPENDED;
public class AzureComputeImageExtension implements ImageExtension {
private final AzureComputeApi api;
private final ListeningExecutorService userExecutor;
private final Supplier<View> blobstore = null;
private final String group;
private final Predicate<URI> imageAvailablePredicate;
private final Predicate<String> nodeSuspendedPredicate;
private final AzureComputeConstants azureComputeConstants;
private final ListeningExecutorService userExecutor;
private final String group;
private final VMImageToImage imageReferenceToImage;
public static final String CONTAINER_NAME = "vhdsnew";
public static final String CONTAINER_NAME = "jclouds";
public static final String CUSTOM_IMAGE_PREFIX = "#";
@Inject
@ -69,29 +73,28 @@ public class AzureComputeImageExtension implements ImageExtension {
this.userExecutor = userExecutor;
this.group = azureComputeConstants.azureResourceGroup();
this.imageReferenceToImage = imageReferenceToImage;
this.api = api;
this.imageAvailablePredicate = imageAvailablePredicate;
this.nodeSuspendedPredicate = nodeSuspendedPredicate;
this.azureComputeConstants = azureComputeConstants;
this.api = api;
}
@Override
public ImageTemplate buildImageTemplateFromNode(String name, String id) {
String imageName = name.toLowerCase();
return new ImageTemplateBuilder.CloneImageTemplateBuilder().nodeId(id).name(imageName).build();
String nameLowerCase = name.toLowerCase();
return new ImageTemplateBuilder.CloneImageTemplateBuilder().nodeId(id).name(nameLowerCase).build();
}
@Override
public ListenableFuture<Image> createImage(ImageTemplate template) {
final CloneImageTemplate cloneTemplate = (CloneImageTemplate) template;
final String id = cloneTemplate.getSourceNodeId();
final String name = cloneTemplate.getName();
final String storageAccountName = id.replaceAll("[^A-Za-z0-9 ]", "") + "stor";
// VM needs to be stopped before it can be generalized
String status = "";
api.getVirtualMachineApi(group).stop(id);
//Poll until resource is ready to be used
if (nodeSuspendedPredicate.apply(id)) {
return userExecutor.submit(new Callable<Image>() {
@Override
@ -118,8 +121,7 @@ public class AzureComputeImageExtension implements ImageExtension {
disks[1] = datadiskObject.get("name");
VirtualMachine vm = api.getVirtualMachineApi(group).get(id);
String location = vm.location();
final VMImage ref = VMImage.create(CUSTOM_IMAGE_PREFIX + group, CUSTOM_IMAGE_PREFIX + name, disks[0], disks[1], location, false);
final VMImage ref = VMImage.create(group, storageAccountName, disks[0], disks[1], name, "custom", vm.location());
return imageReferenceToImage.apply(ref);
}
}
@ -138,6 +140,17 @@ public class AzureComputeImageExtension implements ImageExtension {
@Override
public boolean deleteImage(String id) {
VMImage image = VMImageToImage.decodeFieldsFromUniqueId(id);
if (image.custom()) {
StorageServiceKeys keys = api.getStorageAccountApi(image.group()).getKeys(image.storage());
// This removes now all the images in this storage. At least in theory, there should be just one and if there is
// more, they should be copies of each other.
BlobHelper.deleteContainerIfExists(image.storage(), keys.key1(), "system");
return !BlobHelper.customImageExists(image.storage(), keys.key1());
}
return false;
}
}

View File

@ -210,7 +210,7 @@ public class DeploymentToNodeMetadata implements Function<VMDeployment, NodeMeta
if (imageReference != null) {
VMImage vmImage = VMImage.create(imageReference.publisher(), imageReference.offer(), imageReference.sku(),
imageReference.version(), locationName, false);
imageReference.version(), locationName);
Image image = vmImageToImage.apply(vmImage);
builder.imageId(image.getId());
}

View File

@ -17,7 +17,6 @@
package org.jclouds.azurecompute.arm.compute.functions;
import static com.google.common.base.Preconditions.checkNotNull;
import static org.jclouds.azurecompute.arm.compute.extensions.AzureComputeImageExtension.CUSTOM_IMAGE_PREFIX;
import static org.jclouds.azurecompute.arm.compute.functions.DeploymentToNodeMetadata.AZURE_LOGIN_PASSWORD;
import static org.jclouds.azurecompute.arm.compute.functions.DeploymentToNodeMetadata.AZURE_LOGIN_USERNAME;
@ -67,8 +66,33 @@ public class VMImageToImage implements Function<VMImage, Image> {
return (imageReference.globallyAvailable() ? "global" : imageReference.location()) + "/" + imageReference.publisher() + "/" + imageReference.offer() + "/" + imageReference.sku();
}
public static String[] decodeFieldsFromUniqueId(final String id) {
return checkNotNull(id, "id").split("/");
public static String encodeFieldsToUniqueIdCustom(VMImage imageReference){
return (imageReference.globallyAvailable() ? "global" : imageReference.location()) + "/" + imageReference.group() + "/" + imageReference.storage() + "/" + imageReference.vhd1() + "/" + imageReference.offer();
}
public static VMImage decodeFieldsFromUniqueId(final String id) {
String fields[] = checkNotNull(id, "id").split("/");
VMImage vmImage;
boolean custom = fields.length == 5;
if (custom) {
/* id fields indexes
0: imageReference.location) + "/" +
1: imageReference.group + "/" +
2: imageReference.storage + "/" +
3: imageReference.vhd1 + "/" +
4: imageReference.offer
*/
vmImage = VMImage.create(fields[1], fields[2], fields[3], null, null, fields[4], fields[0]);
} else {
/* id fields indexes
0: imageReference.location) + "/" +
1: imageReference.publisher + "/" +
2: imageReference.offer + "/" +
3: imageReference.sku + "/" +
*/
vmImage = VMImage.create(fields[1], fields[2], fields[3], null, fields[0]);
}
return vmImage;
}
@Inject
@ -80,14 +104,28 @@ public class VMImageToImage implements Function<VMImage, Image> {
public Image apply(final VMImage image) {
Credentials credentials = new Credentials(AZURE_LOGIN_USERNAME, AZURE_LOGIN_PASSWORD);
String name = "";
if (image.offer().startsWith(CUSTOM_IMAGE_PREFIX)) {
name = image.offer().substring(CUSTOM_IMAGE_PREFIX.length());
} else {
name = image.offer();
}
if (image.custom()) {
final ImageBuilder builder = new ImageBuilder()
.name(name)
.location(FluentIterable.from(locations.get())
.firstMatch(LocationPredicates.idEquals(image.location()))
.get())
.name(image.name())
.description("#" + image.group())
.status(Image.Status.AVAILABLE)
.version(image.storage())
.providerId(image.vhd1())
.id(encodeFieldsToUniqueIdCustom(image))
.defaultCredentials(LoginCredentials.fromCredentials(credentials));
final OperatingSystem.Builder osBuilder = osFamily().apply(image);
Image retimage = builder.operatingSystem(osBuilder.build()).build();
return retimage;
}
else {
final ImageBuilder builder = new ImageBuilder()
.name(image.offer())
.description(image.sku())
.status(Image.Status.AVAILABLE)
.version(image.sku())
@ -101,6 +139,7 @@ public class VMImageToImage implements Function<VMImage, Image> {
final OperatingSystem.Builder osBuilder = osFamily().apply(image);
return builder.operatingSystem(osBuilder.build()).build();
}
}
public static Function<VMImage, OperatingSystem.Builder> osFamily() {
return new Function<VMImage, OperatingSystem.Builder>() {
@ -124,12 +163,16 @@ public class VMImageToImage implements Function<VMImage, Image> {
family = OsFamily.OEL;
}
String sku = image.sku();
if (image.custom())
sku = image.vhd1();
// only 64bit OS images are supported by Azure ARM
return OperatingSystem.builder().
family(family).
is64Bit(true).
description(image.sku()).
version(image.sku());
description(sku).
version(sku);
}
};
}

View File

@ -17,34 +17,39 @@
package org.jclouds.azurecompute.arm.domain;
import com.google.auto.value.AutoValue;
import org.jclouds.javax.annotation.Nullable;
import org.jclouds.json.SerializedNames;
@AutoValue
public abstract class VMImage {
/**
* The publisher of the image reference.
*/
@Nullable
public abstract String publisher();
/**
* The offer of the image reference.
*/
@Nullable
public abstract String offer();
/**
* The sku of the image reference.
*/
@Nullable
public abstract String sku();
/**
* The version of the image reference.
*/
@Nullable
public abstract String version();
/**
* The location from where Image was fetched
*/
@Nullable
public abstract String location();
/**
@ -52,9 +57,50 @@ public abstract class VMImage {
*/
public abstract boolean globallyAvailable();
@SerializedNames({ "publisher", "offer", "sku", "version", "location", "globallyAvailable"})
public static VMImage create(String publisher, String offer, String sku, String version, String location, boolean globallyAvailable) {
/**
* The group of the custom image
*/
@Nullable
public abstract String group();
return new AutoValue_VMImage(publisher, offer, sku, version, location, globallyAvailable);
/**
* The storage of the custom image.
*/
@Nullable
public abstract String storage();
/**
* The vhd1 of the custom image
*/
@Nullable
public abstract String vhd1();
/**
* The vhd2 of the custom image.
*/
@Nullable
public abstract String vhd2();
/**
* The name of the custom image template.
*/
@Nullable
public abstract String name();
/**
* True if custom image
*/
public abstract boolean custom();
@SerializedNames({ "publisher", "offer", "sku", "version", "location"})
public static VMImage create(String publisher, String offer, String sku, String version, String location) {
return new AutoValue_VMImage(publisher, offer, sku, version, location, false, null, null, null, null, null, false);
}
@SerializedNames({ "group", "storage", "vhd1", "vhd2", "name", "offer", "location"})
public static VMImage create(String group, String storage, String vhd1, String vhd2, String name, String offer, String location) {
return new AutoValue_VMImage(null, offer, null, null, location, false, group, storage, vhd1, vhd2, name, true);
}
}

View File

@ -0,0 +1,91 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jclouds.azurecompute.arm.util;
import org.jclouds.ContextBuilder;
import org.jclouds.azure.storage.domain.BoundedSet;
import org.jclouds.azureblob.AzureBlobClient;
import org.jclouds.azureblob.domain.BlobProperties;
import org.jclouds.azureblob.domain.ContainerProperties;
import org.jclouds.azureblob.domain.ListBlobsResponse;
import org.jclouds.azurecompute.arm.domain.VMImage;
import org.jclouds.util.Closeables2;
import java.util.ArrayList;
import java.util.List;
public class BlobHelper {
public static void deleteContainerIfExists(String storage, String key, String containerName) {
final AzureBlobClient azureBlob = ContextBuilder.newBuilder("azureblob")
.credentials(storage, key)
.buildApi(AzureBlobClient.class);
try {
azureBlob.deleteContainer(containerName);
}
finally {
Closeables2.closeQuietly(azureBlob);
}
}
public static boolean customImageExists(String storage, String key) {
final AzureBlobClient azureBlob = ContextBuilder.newBuilder("azureblob")
.credentials(storage, key)
.buildApi(AzureBlobClient.class);
try {
return azureBlob.containerExists("system");
}
finally {
Closeables2.closeQuietly(azureBlob);
}
}
public static List<VMImage> getImages(String containerName, String group,
String storageAccountName, String key, String offer, String location) {
final AzureBlobClient azureBlob = ContextBuilder.newBuilder("azureblob")
.credentials(storageAccountName, key)
.buildApi(AzureBlobClient.class);
List<VMImage> list = new ArrayList<VMImage>();
try {
BoundedSet<ContainerProperties> containerList = azureBlob.listContainers();
for (ContainerProperties props : containerList) {
if (props.getName().equals("system")) {
ListBlobsResponse blobList = azureBlob.listBlobs("system");
String osDisk = "";
String dataDisk = "";
for (BlobProperties blob : blobList) {
String name = blob.getName();
if (dataDisk.length() == 0) dataDisk = name.substring(1 + name.lastIndexOf('/'));
else if (osDisk.length() == 0) osDisk = name.substring(1 + name.lastIndexOf('/'));
}
final VMImage ref = VMImage.create(group, storageAccountName, osDisk, dataDisk, "test-create-image", "custom", location);
list.add(ref);
}
}
}
finally {
Closeables2.closeQuietly(azureBlob);
}
return list;
}
}

View File

@ -182,7 +182,7 @@ public class DeploymentTemplateBuilder {
String imageName = template.getImage().getName();
if (imageName.startsWith(CUSTOM_IMAGE_PREFIX)) {
storageAccountName = imageName.substring(CUSTOM_IMAGE_PREFIX.length()); // get group name
storageAccountName = template.getImage().getVersion();
}
if (Strings.isNullOrEmpty(storageAccountName)) {
@ -414,13 +414,12 @@ public class DeploymentTemplateBuilder {
boolean usingMarketplaceImage = true;
String cusotomImageUri = "";
// TODO: make new fields for group information
String publisher = template.getImage().getProviderId();
String storageName = template.getImage().getName();
String sku = template.getImage().getDescription(); // this is actual VHD
if (storageName.startsWith(CUSTOM_IMAGE_PREFIX)) {
storageName = storageName.substring(CUSTOM_IMAGE_PREFIX.length()); // get group name
cusotomImageUri = sku;
// Handle custom image case if description starts with CUSTOM_IMAGE_PREFIX
String vhd1 = template.getImage().getProviderId();
String description = template.getImage().getDescription();
if (description.substring(0, CUSTOM_IMAGE_PREFIX.length()).equals(CUSTOM_IMAGE_PREFIX)) {
String storageName = template.getImage().getVersion();
cusotomImageUri = vhd1;
cusotomImageUri = "https://" + storageName + ".blob.core.windows.net/system/Microsoft.Compute/Images/" + AzureComputeImageExtension.CONTAINER_NAME + "/" + cusotomImageUri;
}

View File

@ -63,7 +63,7 @@ public class AzureComputeImageExtensionLiveTest extends BaseImageExtensionLiveTe
properties.setProperty(TIMEOUT_PORT_OPEN, scriptTimeout + "");
properties.setProperty(TIMEOUT_NODE_TERMINATED, scriptTimeout + "");
properties.setProperty(TIMEOUT_NODE_SUSPENDED, scriptTimeout + "");
properties.put(RESOURCE_GROUP_NAME, "a5");
properties.put(RESOURCE_GROUP_NAME, "jcloudsgroup");
properties.put(ComputeServiceProperties.POLL_INITIAL_PERIOD, 1000);
properties.put(ComputeServiceProperties.POLL_MAX_PERIOD, 10000);
@ -86,4 +86,5 @@ public class AzureComputeImageExtensionLiveTest extends BaseImageExtensionLiveTe
return pm;
}
}