honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Use Azure Storage 2.0.0 

Microsoft team has released a new specific project to deal with storage with a much cleaner API than the previous version.

See https://github.com/azure/azure-storage-java
Documentation is here: http://azure.microsoft.com/en-us/documentation/articles/storage-java-how-to-use-blob-storage/

Note that the produced ZIP file has been reduced from 5mb to 1.3mb.

Related to #38

(cherry picked from commit 4467254)
(cherry picked from commit b2f1e4d)
This commit is contained in:
David Pilato 2015-02-04 17:11:56 +01:00
parent c20371aa8f
commit 54de108a51
10 changed files with 51 additions and 82 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
pom.xml
View File

@ -68,15 +68,9 @@ governing permissions and limitations under the License. -->
<!-- Azure API -->
<dependency>
<groupId>com.microsoft.windowsazure</groupId>
<artifactId>microsoft-windowsazure-api</artifactId>
<version>0.4.6</version>
<exclusions>
<exclusion>
<groupId>javax.xml.stream</groupId>
<artifactId>stax-api</artifactId>
</exclusion>
</exclusions>
<groupId>com.microsoft.azure</groupId>
<artifactId>azure-storage</artifactId>
<version>2.0.0</version>
</dependency>
<dependency>

2
src/main/assemblies/plugin.xml
View File

@ -30,7 +30,7 @@ governing permissions and limitations under the License. -->
<useProjectArtifact>true</useProjectArtifact>
<useTransitiveFiltering>true</useTransitiveFiltering>
<includes>
<include>com.microsoft.windowsazure:microsoft-windowsazure-api</include>
<include>com.microsoft.azure:azure-storage</include>
</includes>
</dependencySet>
</dependencySets>

9
src/main/java/org/elasticsearch/cloud/azure/AzureStorageService.java
View File

@ -19,8 +19,7 @@
package org.elasticsearch.cloud.azure;
import com.microsoft.windowsazure.services.core.ServiceException;
import com.microsoft.windowsazure.services.core.storage.StorageException;
import com.microsoft.azure.storage.StorageException;
import org.elasticsearch.common.blobstore.BlobMetaData;
import org.elasticsearch.common.collect.ImmutableMap;
@ -48,17 +47,17 @@ public interface AzureStorageService {
void createContainer(String container) throws URISyntaxException, StorageException;
void deleteFiles(String container, String path) throws URISyntaxException, StorageException, ServiceException;
void deleteFiles(String container, String path) throws URISyntaxException, StorageException;
boolean blobExists(String container, String blob) throws URISyntaxException, StorageException;
void deleteBlob(String container, String blob) throws URISyntaxException, StorageException;
InputStream getInputStream(String container, String blob) throws ServiceException;
InputStream getInputStream(String container, String blob) throws URISyntaxException, StorageException;
OutputStream getOutputStream(String container, String blob) throws URISyntaxException, StorageException;
ImmutableMap<String,BlobMetaData> listBlobsByPrefix(String container, String keyPath, String prefix) throws URISyntaxException, StorageException, ServiceException;
ImmutableMap<String,BlobMetaData> listBlobsByPrefix(String container, String keyPath, String prefix) throws URISyntaxException, StorageException;
void moveBlob(String container, String sourceBlob, String targetBlob) throws URISyntaxException, StorageException;
}

67
src/main/java/org/elasticsearch/cloud/azure/AzureStorageServiceImpl.java
View File

@ -19,21 +19,9 @@
package org.elasticsearch.cloud.azure;
import com.microsoft.windowsazure.services.blob.BlobConfiguration;
import com.microsoft.windowsazure.services.blob.BlobContract;
import com.microsoft.windowsazure.services.blob.BlobService;
import com.microsoft.windowsazure.services.blob.client.CloudBlobClient;
import com.microsoft.windowsazure.services.blob.client.CloudBlobContainer;
import com.microsoft.windowsazure.services.blob.client.CloudBlockBlob;
import com.microsoft.windowsazure.services.blob.client.ListBlobItem;
import com.microsoft.windowsazure.services.blob.models.BlobProperties;
import com.microsoft.windowsazure.services.blob.models.GetBlobResult;
import com.microsoft.windowsazure.services.blob.models.ListBlobsOptions;
import com.microsoft.windowsazure.services.blob.models.ListBlobsResult;
import com.microsoft.windowsazure.services.core.Configuration;
import com.microsoft.windowsazure.services.core.ServiceException;
import com.microsoft.windowsazure.services.core.storage.CloudStorageAccount;
import com.microsoft.windowsazure.services.core.storage.StorageException;
import com.microsoft.azure.storage.CloudStorageAccount;
import com.microsoft.azure.storage.StorageException;
import com.microsoft.azure.storage.blob.*;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.common.blobstore.BlobMetaData;
import org.elasticsearch.common.blobstore.support.PlainBlobMetaData;
@ -48,7 +36,6 @@ import java.io.InputStream;
import java.io.OutputStream;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.List;
/**
*
@ -60,10 +47,7 @@ public class AzureStorageServiceImpl extends AbstractLifecycleComponent<AzureSto
private final String key;
private final String blob;
private CloudStorageAccount storage_account;
private CloudBlobClient client;
private BlobContract service;
@Inject
public AzureStorageServiceImpl(Settings settings, SettingsFilter settingsFilter) {
@ -84,14 +68,11 @@ public class AzureStorageServiceImpl extends AbstractLifecycleComponent<AzureSto
+ "AccountName="+ account +";"
+ "AccountKey=" + key;
Configuration configuration = Configuration.getInstance();
configuration.setProperty(BlobConfiguration.ACCOUNT_NAME, account);
configuration.setProperty(BlobConfiguration.ACCOUNT_KEY, key);
configuration.setProperty(BlobConfiguration.URI, blob);
service = BlobService.create(configuration);
// Retrieve storage account from connection-string.
CloudStorageAccount storageAccount = CloudStorageAccount.parse(storageConnectionString);
storage_account = CloudStorageAccount.parse(storageConnectionString);
client = storage_account.createCloudBlobClient();
// Create the blob client.
client = storageAccount.createCloudBlobClient();
}
} catch (Exception e) {
// Can not start Azure Storage Client
@ -129,7 +110,7 @@ public class AzureStorageServiceImpl extends AbstractLifecycleComponent<AzureSto
try {
CloudBlobContainer blob_container = client.getContainerReference(container);
logger.trace("creating container [{}]", container);
blob_container.createIfNotExist();
blob_container.createIfNotExists();
} catch (IllegalArgumentException e) {
logger.trace("fails creating container [{}]", container, e.getMessage());
throw new RepositoryException(container, e.getMessage());
@ -137,19 +118,15 @@ public class AzureStorageServiceImpl extends AbstractLifecycleComponent<AzureSto
}
@Override
public void deleteFiles(String container, String path) throws URISyntaxException, StorageException, ServiceException {
public void deleteFiles(String container, String path) throws URISyntaxException, StorageException {
logger.trace("delete files container [{}], path [{}]", container, path);
// Container name must be lower case.
CloudBlobContainer blob_container = client.getContainerReference(container);
if (blob_container.exists()) {
ListBlobsOptions options = new ListBlobsOptions();
options.setPrefix(path);
List<ListBlobsResult.BlobEntry> blobs = service.listBlobs(container, options).getBlobs();
for (ListBlobsResult.BlobEntry blob : blobs) {
logger.trace("removing in container [{}], path [{}], blob [{}]", container, path, blob.getName());
service.deleteBlob(container, blob.getName());
for (ListBlobItem blobItem : blob_container.listBlobs(path)) {
logger.trace("removing blob [{}]", blobItem.getUri());
deleteBlob(container, blobItem.getUri().toString());
}
}
}
@ -180,10 +157,9 @@ public class AzureStorageServiceImpl extends AbstractLifecycleComponent<AzureSto
}
@Override
public InputStream getInputStream(String container, String blob) throws ServiceException {
public InputStream getInputStream(String container, String blob) throws URISyntaxException, StorageException {
logger.trace("reading container [{}], blob [{}]", container, blob);
GetBlobResult blobResult = service.getBlob(container, blob);
return blobResult.getContentStream();
return client.getContainerReference(container).getBlockBlobReference(blob).openInputStream();
}
@Override
@ -193,21 +169,20 @@ public class AzureStorageServiceImpl extends AbstractLifecycleComponent<AzureSto
}
@Override
public ImmutableMap<String, BlobMetaData> listBlobsByPrefix(String container, String keyPath, String prefix) throws URISyntaxException, StorageException, ServiceException {
public ImmutableMap<String, BlobMetaData> listBlobsByPrefix(String container, String keyPath, String prefix) throws URISyntaxException, StorageException {
logger.debug("listing container [{}], keyPath [{}], prefix [{}]", container, keyPath, prefix);
ImmutableMap.Builder<String, BlobMetaData> blobsBuilder = ImmutableMap.builder();
CloudBlobContainer blob_container = client.getContainerReference(container);
if (blob_container.exists()) {
Iterable<ListBlobItem> blobs = blob_container.listBlobs(keyPath + prefix);
for (ListBlobItem blob : blobs) {
URI uri = blob.getUri();
for (ListBlobItem blobItem : blob_container.listBlobs(keyPath + prefix)) {
URI uri = blobItem.getUri();
logger.trace("blob url [{}]", uri);
String blobpath = uri.getPath().substring(container.length() + 1);
BlobProperties properties = service.getBlobProperties(container, blobpath).getProperties();
BlobProperties properties = blob_container.getBlockBlobReference(blobpath).getProperties();
String name = blobpath.substring(keyPath.length() + 1);
logger.trace("blob url [{}], name [{}], size [{}]", uri, name, properties.getContentLength());
blobsBuilder.put(name, new PlainBlobMetaData(name, properties.getContentLength()));
logger.trace("blob url [{}], name [{}], size [{}]", uri, name, properties.getLength());
blobsBuilder.put(name, new PlainBlobMetaData(name, properties.getLength()));
}
}
@ -221,7 +196,7 @@ public class AzureStorageServiceImpl extends AbstractLifecycleComponent<AzureSto
CloudBlockBlob blobSource = blob_container.getBlockBlobReference(sourceBlob);
if (blobSource.exists()) {
CloudBlockBlob blobTarget = blob_container.getBlockBlobReference(targetBlob);
blobTarget.copyFromBlob(blobSource);
blobTarget.startCopyFromBlob(blobSource);
blobSource.delete();
logger.debug("moveBlob container [{}], sourceBlob [{}], targetBlob [{}] -> done", container, sourceBlob, targetBlob);
}

9
src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobContainer.java
View File

@ -19,8 +19,7 @@
package org.elasticsearch.cloud.azure.blobstore;
import com.microsoft.windowsazure.services.core.ServiceException;
import com.microsoft.windowsazure.services.core.storage.StorageException;
import com.microsoft.azure.storage.StorageException;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.blobstore.BlobMetaData;
import org.elasticsearch.common.blobstore.BlobPath;
@ -73,11 +72,13 @@ public class AzureBlobContainer extends AbstractBlobContainer {
public InputStream openInput(String blobName) throws IOException {
try {
return blobStore.client().getInputStream(blobStore.container(), buildKey(blobName));
} catch (ServiceException e) {
} catch (StorageException e) {
if (e.getHttpStatusCode() == HttpURLConnection.HTTP_NOT_FOUND) {
throw new FileNotFoundException(e.getMessage());
}
throw new IOException(e);
} catch (URISyntaxException e) {
throw new IOException(e);
}
}
@ -112,7 +113,7 @@ public class AzureBlobContainer extends AbstractBlobContainer {
try {
return blobStore.client().listBlobsByPrefix(blobStore.container(), keyPath, prefix);
} catch (URISyntaxException | StorageException | ServiceException e) {
} catch (URISyntaxException | StorageException e) {
logger.warn("can not access [{}] in container {{}}: {}", prefix, blobStore.container(), e.getMessage());
throw new IOException(e);
}

5
src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobStore.java
View File

@ -19,8 +19,7 @@
package org.elasticsearch.cloud.azure.blobstore;
import com.microsoft.windowsazure.services.core.ServiceException;
import com.microsoft.windowsazure.services.core.storage.StorageException;
import com.microsoft.azure.storage.StorageException;
import org.elasticsearch.cloud.azure.AzureStorageService;
import org.elasticsearch.common.blobstore.BlobContainer;
import org.elasticsearch.common.blobstore.BlobPath;
@ -81,7 +80,7 @@ public class AzureBlobStore extends AbstractComponent implements BlobStore {
try {
client.deleteFiles(container, keyPath);
} catch (URISyntaxException | StorageException | ServiceException e) {
} catch (URISyntaxException | StorageException e) {
logger.warn("can not remove [{}] in container {{}}: {}", keyPath, container, e.getMessage());
}
}

2
src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java
View File

@ -19,7 +19,7 @@
package org.elasticsearch.repositories.azure;
import com.microsoft.windowsazure.services.core.storage.StorageException;
import com.microsoft.azure.storage.StorageException;
import org.elasticsearch.cloud.azure.AzureStorageService;
import org.elasticsearch.cloud.azure.blobstore.AzureBlobStore;
import org.elasticsearch.cluster.metadata.MetaData;

7
src/test/java/org/elasticsearch/repositories/azure/AbstractAzureRepositoryServiceTest.java
View File

@ -19,8 +19,7 @@
package org.elasticsearch.repositories.azure;
import com.microsoft.windowsazure.services.core.ServiceException;
import com.microsoft.windowsazure.services.core.storage.StorageException;
import com.microsoft.azure.storage.StorageException;
import org.elasticsearch.cloud.azure.AbstractAzureTest;
import org.elasticsearch.cloud.azure.AzureStorageService;
import org.elasticsearch.cluster.metadata.IndexMetaData;
@ -87,7 +86,7 @@ public abstract class AbstractAzureRepositoryServiceTest extends AbstractAzureTe
}
@Before @After
public final void wipe() throws StorageException, ServiceException, URISyntaxException {
public final void wipe() throws StorageException, URISyntaxException {
wipeRepositories();
cleanRepositoryFiles(basePath);
}
@ -95,7 +94,7 @@ public abstract class AbstractAzureRepositoryServiceTest extends AbstractAzureTe
/**
* Purge the test container
*/
public void cleanRepositoryFiles(String path) throws StorageException, ServiceException, URISyntaxException {
public void cleanRepositoryFiles(String path) throws StorageException, URISyntaxException {
String container = internalCluster().getInstance(Settings.class).get("repositories.azure.container");
logger.info("--> remove blobs in container [{}]", container);
AzureStorageService client = internalCluster().getInstance(AzureStorageService.class);

13
src/test/java/org/elasticsearch/repositories/azure/AzureSnapshotRestoreITest.java
View File

@ -20,8 +20,7 @@
package org.elasticsearch.repositories.azure;
import com.microsoft.windowsazure.services.core.ServiceException;
import com.microsoft.windowsazure.services.core.storage.StorageException;
import com.microsoft.azure.storage.StorageException;
import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryResponse;
import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse;
import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse;
@ -83,7 +82,7 @@ public class AzureSnapshotRestoreITest extends AbstractAzureTest {
}
@Before
public final void wipeBefore() throws StorageException, ServiceException, URISyntaxException {
public final void wipeBefore() throws StorageException, URISyntaxException {
wipeRepositories();
cleanRepositoryFiles(
getContainerName(),
@ -92,7 +91,7 @@ public class AzureSnapshotRestoreITest extends AbstractAzureTest {
}
@After
public final void wipeAfter() throws StorageException, ServiceException, URISyntaxException {
public final void wipeAfter() throws StorageException, URISyntaxException {
wipeRepositories();
cleanRepositoryFiles(
getContainerName(),
@ -243,7 +242,7 @@ public class AzureSnapshotRestoreITest extends AbstractAzureTest {
* For issue #26: https://github.com/elasticsearch/elasticsearch-cloud-azure/issues/26
*/
@Test
public void testListBlobs_26() throws StorageException, ServiceException, URISyntaxException {
public void testListBlobs_26() throws StorageException, URISyntaxException {
createIndex("test-idx-1", "test-idx-2", "test-idx-3");
ensureGreen();
@ -302,7 +301,7 @@ public class AzureSnapshotRestoreITest extends AbstractAzureTest {
* For issue #28: https://github.com/elasticsearch/elasticsearch-cloud-azure/issues/28
*/
@Test
public void testGetDeleteNonExistingSnapshot_28() throws StorageException, ServiceException, URISyntaxException {
public void testGetDeleteNonExistingSnapshot_28() throws StorageException, URISyntaxException {
ClusterAdminClient client = client().admin().cluster();
logger.info("--> creating azure repository without any path");
PutRepositoryResponse putRepositoryResponse = client.preparePutRepository("test-repo").setType("azure")
@ -452,7 +451,7 @@ public class AzureSnapshotRestoreITest extends AbstractAzureTest {
/**
* Purge the test containers
*/
public void cleanRepositoryFiles(String... containers) throws StorageException, ServiceException, URISyntaxException {
public void cleanRepositoryFiles(String... containers) throws StorageException, URISyntaxException {
AzureStorageService client = internalCluster().getInstance(AzureStorageService.class);
for (String container : containers) {
logger.info("--> remove container [{}]", container);

7
src/test/java/org/elasticsearch/repositories/azure/AzureStorageServiceMock.java
View File

@ -19,7 +19,7 @@
package org.elasticsearch.repositories.azure;
import com.microsoft.windowsazure.services.core.storage.StorageException;
import com.microsoft.azure.storage.StorageException;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.cloud.azure.AzureStorageService;
import org.elasticsearch.common.blobstore.BlobMetaData;
@ -29,7 +29,10 @@ import org.elasticsearch.common.component.AbstractLifecycleComponent;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings;
import java.io.*;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URISyntaxException;
import java.util.Locale;
import java.util.Map;