mirror of https://github.com/apache/nifi.git
NIFI-1833 - Azure Storage processors
Signed-off-by: Bryan Rosander <brosander@apache.org>
This commit is contained in:
Simon Elliston Ball
Bryan Rosander
parent
7d242076ce
commit
3488a169ca
|
|
@ -35,6 +35,12 @@
|
||||||
<artifactId>nifi-azure-processors</artifactId>
|
<artifactId>nifi-azure-processors</artifactId>
|
||||||
<version>1.2.0-SNAPSHOT</version>
|
<version>1.2.0-SNAPSHOT</version>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.nifi</groupId>
|
||||||
|
<artifactId>nifi-standard-services-api-nar</artifactId>
|
||||||
|
<type>nar</type>
|
||||||
|
</dependency>
|
||||||
</dependencies>
|
</dependencies>
|
||||||
|
|
||||||
</project>
|
</project>
|
||||||
|
|
|
||||||
|
|
@ -31,11 +31,35 @@ language governing permissions and limitations under the License. -->
|
||||||
<groupId>org.apache.nifi</groupId>
|
<groupId>org.apache.nifi</groupId>
|
||||||
<artifactId>nifi-utils</artifactId>
|
<artifactId>nifi-utils</artifactId>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.nifi</groupId>
|
||||||
|
<artifactId>nifi-ssl-context-service-api</artifactId>
|
||||||
|
<scope>provided</scope>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.avro</groupId>
|
||||||
|
<artifactId>avro</artifactId>
|
||||||
|
</dependency>
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>com.microsoft.azure</groupId>
|
<groupId>com.microsoft.azure</groupId>
|
||||||
<artifactId>azure-eventhubs</artifactId>
|
<artifactId>azure-eventhubs</artifactId>
|
||||||
<version>0.9.0</version>
|
<version>0.9.0</version>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>com.fasterxml.jackson.core</groupId>
|
||||||
|
<artifactId>jackson-core</artifactId>
|
||||||
|
<version>2.8.6</version>
|
||||||
|
</dependency>
|
||||||
|
<!--<dependency>
|
||||||
|
<groupId>com.microsoft.eventhubs.client</groupId>
|
||||||
|
<artifactId>eventhubs-client</artifactId>
|
||||||
|
<version>0.9.1</version>
|
||||||
|
</dependency>-->
|
||||||
|
<dependency>
|
||||||
|
<groupId>com.microsoft.azure</groupId>
|
||||||
|
<artifactId>azure-storage</artifactId>
|
||||||
|
<version>5.0.0</version>
|
||||||
|
</dependency>
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.apache.nifi</groupId>
|
<groupId>org.apache.nifi</groupId>
|
||||||
<artifactId>nifi-mock</artifactId>
|
<artifactId>nifi-mock</artifactId>
|
||||||
|
|
@ -57,5 +81,10 @@ language governing permissions and limitations under the License. -->
|
||||||
<version>${powermock.version}</version>
|
<version>${powermock.version}</version>
|
||||||
<scope>test</scope>
|
<scope>test</scope>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.nifi</groupId>
|
||||||
|
<artifactId>nifi-standard-processors</artifactId>
|
||||||
|
<version>${project.version}</version>
|
||||||
|
</dependency>
|
||||||
</dependencies>
|
</dependencies>
|
||||||
</project>
|
</project>
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,39 @@
|
||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
* contributor license agreements. See the NOTICE file distributed with
|
||||||
|
* this work for additional information regarding copyright ownership.
|
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
* (the "License"); you may not use this file except in compliance with
|
||||||
|
* the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.nifi.processors.azure;
|
||||||
|
|
||||||
|
import org.apache.nifi.components.PropertyDescriptor;
|
||||||
|
import org.apache.nifi.processor.util.StandardValidators;
|
||||||
|
|
||||||
|
import java.util.Arrays;
|
||||||
|
import java.util.Collections;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
public abstract class AbstractAzureBlobProcessor extends AbstractAzureProcessor {
|
||||||
|
|
||||||
|
public static final PropertyDescriptor BLOB = new PropertyDescriptor.Builder().name("Blob").description("The filename of the blob").addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
|
||||||
|
.expressionLanguageSupported(true).required(true).defaultValue("${azure.blobname}").build();
|
||||||
|
|
||||||
|
public static final List<PropertyDescriptor> properties = Collections
|
||||||
|
.unmodifiableList(Arrays.asList(AzureConstants.ACCOUNT_NAME, AzureConstants.ACCOUNT_KEY, AzureConstants.CONTAINER, BLOB));
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
|
||||||
|
return properties;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,85 @@
|
||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
* contributor license agreements. See the NOTICE file distributed with
|
||||||
|
* this work for additional information regarding copyright ownership.
|
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
* (the "License"); you may not use this file except in compliance with
|
||||||
|
* the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.nifi.processors.azure;
|
||||||
|
|
||||||
|
import java.net.URISyntaxException;
|
||||||
|
import java.security.InvalidKeyException;
|
||||||
|
import java.util.Arrays;
|
||||||
|
import java.util.Collections;
|
||||||
|
import java.util.HashSet;
|
||||||
|
import java.util.Set;
|
||||||
|
|
||||||
|
import org.apache.nifi.flowfile.FlowFile;
|
||||||
|
import org.apache.nifi.processor.AbstractProcessor;
|
||||||
|
import org.apache.nifi.processor.ProcessContext;
|
||||||
|
import org.apache.nifi.processor.Relationship;
|
||||||
|
|
||||||
|
import com.microsoft.azure.storage.CloudStorageAccount;
|
||||||
|
|
||||||
|
public abstract class AbstractAzureProcessor extends AbstractProcessor {
|
||||||
|
|
||||||
|
public static final Relationship REL_SUCCESS = new Relationship.Builder().name("success").description("All FlowFiles that are received are routed to success").build();
|
||||||
|
protected static final Relationship REL_FAILURE = new Relationship.Builder().name("failure").description("Any failed fetches will be transferred to the failure relation.").build();
|
||||||
|
public static final Set<Relationship> RELATIONSHIPS = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(REL_SUCCESS, REL_FAILURE)));
|
||||||
|
|
||||||
|
protected CloudStorageAccount createStorageConnection(ProcessContext context) {
|
||||||
|
final String accountName = context.getProperty(AzureConstants.ACCOUNT_NAME).evaluateAttributeExpressions().getValue();
|
||||||
|
final String accountKey = context.getProperty(AzureConstants.ACCOUNT_KEY).evaluateAttributeExpressions().getValue();
|
||||||
|
return createStorageConnectionFromNameAndKey(accountName, accountKey);
|
||||||
|
}
|
||||||
|
|
||||||
|
protected CloudStorageAccount createStorageConnection(ProcessContext context, FlowFile flowFile) {
|
||||||
|
final String accountName = context.getProperty(AzureConstants.ACCOUNT_NAME).evaluateAttributeExpressions(flowFile).getValue();
|
||||||
|
final String accountKey = context.getProperty(AzureConstants.ACCOUNT_KEY).evaluateAttributeExpressions(flowFile).getValue();
|
||||||
|
return createStorageConnectionFromNameAndKey(accountName, accountKey);
|
||||||
|
}
|
||||||
|
|
||||||
|
private CloudStorageAccount createStorageConnectionFromNameAndKey(String accountName, String accountKey) {
|
||||||
|
final String storageConnectionString = String.format("DefaultEndpointsProtocol=https;AccountName=%s;AccountKey=%s", accountName, accountKey);
|
||||||
|
try {
|
||||||
|
return createStorageAccountFromConnectionString(storageConnectionString);
|
||||||
|
} catch (InvalidKeyException | IllegalArgumentException | URISyntaxException e) {
|
||||||
|
throw new IllegalArgumentException(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Validates the connection string and returns the storage account. The connection string must be in the Azure connection string format.
|
||||||
|
*
|
||||||
|
* @param storageConnectionString
|
||||||
|
* Connection string for the storage service or the emulator
|
||||||
|
* @return The newly created CloudStorageAccount object
|
||||||
|
*
|
||||||
|
*/
|
||||||
|
protected static CloudStorageAccount createStorageAccountFromConnectionString(String storageConnectionString) throws IllegalArgumentException, URISyntaxException, InvalidKeyException {
|
||||||
|
CloudStorageAccount storageAccount;
|
||||||
|
try {
|
||||||
|
storageAccount = CloudStorageAccount.parse(storageConnectionString);
|
||||||
|
} catch (IllegalArgumentException | URISyntaxException e) {
|
||||||
|
throw e;
|
||||||
|
} catch (InvalidKeyException e) {
|
||||||
|
throw e;
|
||||||
|
}
|
||||||
|
return storageAccount;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Set<Relationship> getRelationships() {
|
||||||
|
return RELATIONSHIPS;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,38 @@
|
||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
* contributor license agreements. See the NOTICE file distributed with
|
||||||
|
* this work for additional information regarding copyright ownership.
|
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
* (the "License"); you may not use this file except in compliance with
|
||||||
|
* the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.nifi.processors.azure;
|
||||||
|
|
||||||
|
import org.apache.nifi.components.PropertyDescriptor;
|
||||||
|
import org.apache.nifi.processor.util.StandardValidators;
|
||||||
|
|
||||||
|
public final class AzureConstants {
|
||||||
|
public static final String BLOCK = "Block";
|
||||||
|
public static final String PAGE = "Page";
|
||||||
|
|
||||||
|
public static final PropertyDescriptor ACCOUNT_KEY = new PropertyDescriptor.Builder().name("Storage Account Key").description("The storage account key")
|
||||||
|
.addValidator(StandardValidators.NON_EMPTY_VALIDATOR).expressionLanguageSupported(true).required(true).sensitive(true).build();
|
||||||
|
|
||||||
|
public static final PropertyDescriptor ACCOUNT_NAME = new PropertyDescriptor.Builder().name("Storage Account Name").description("The storage account name")
|
||||||
|
.addValidator(StandardValidators.NON_EMPTY_VALIDATOR).expressionLanguageSupported(true).required(true).sensitive(true).build();
|
||||||
|
|
||||||
|
public static final PropertyDescriptor CONTAINER = new PropertyDescriptor.Builder().name("Container name").description("Name of the azure storage container")
|
||||||
|
.addValidator(StandardValidators.NON_EMPTY_VALIDATOR).expressionLanguageSupported(true).required(true).build();
|
||||||
|
|
||||||
|
private AzureConstants() {
|
||||||
|
// do not instantiate
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,114 @@
|
||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
* contributor license agreements. See the NOTICE file distributed with
|
||||||
|
* this work for additional information regarding copyright ownership.
|
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
* (the "License"); you may not use this file except in compliance with
|
||||||
|
* the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.nifi.processors.azure.storage;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.io.OutputStream;
|
||||||
|
import java.net.URISyntaxException;
|
||||||
|
import java.util.Arrays;
|
||||||
|
import java.util.Collections;
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.concurrent.TimeUnit;
|
||||||
|
|
||||||
|
import org.apache.nifi.annotation.behavior.InputRequirement;
|
||||||
|
import org.apache.nifi.annotation.behavior.WritesAttribute;
|
||||||
|
import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
|
||||||
|
import org.apache.nifi.annotation.behavior.WritesAttributes;
|
||||||
|
import org.apache.nifi.annotation.documentation.CapabilityDescription;
|
||||||
|
import org.apache.nifi.annotation.documentation.Tags;
|
||||||
|
import org.apache.nifi.components.PropertyDescriptor;
|
||||||
|
import org.apache.nifi.flowfile.FlowFile;
|
||||||
|
import org.apache.nifi.processor.ProcessContext;
|
||||||
|
import org.apache.nifi.processor.ProcessSession;
|
||||||
|
import org.apache.nifi.processor.exception.ProcessException;
|
||||||
|
import org.apache.nifi.processor.io.OutputStreamCallback;
|
||||||
|
import org.apache.nifi.processors.azure.AbstractAzureBlobProcessor;
|
||||||
|
import org.apache.nifi.processors.azure.AzureConstants;
|
||||||
|
|
||||||
|
import com.microsoft.azure.storage.CloudStorageAccount;
|
||||||
|
import com.microsoft.azure.storage.StorageException;
|
||||||
|
import com.microsoft.azure.storage.blob.CloudBlob;
|
||||||
|
import com.microsoft.azure.storage.blob.CloudBlobClient;
|
||||||
|
import com.microsoft.azure.storage.blob.CloudBlobContainer;
|
||||||
|
|
||||||
|
@Tags({ "azure", "microsoft", "cloud", "storage", "blob" })
|
||||||
|
@CapabilityDescription("Retrieves contents of an Azure Storage Blob, writing the contents to the content of the FlowFile")
|
||||||
|
@InputRequirement(Requirement.INPUT_REQUIRED)
|
||||||
|
@WritesAttributes({
|
||||||
|
@WritesAttribute(attribute = "azure.length", description = "The length of the blob fetched")
|
||||||
|
})
|
||||||
|
public class FetchAzureBlobStorage extends AbstractAzureBlobProcessor {
|
||||||
|
public static final List<PropertyDescriptor> PROPERTIES = Collections
|
||||||
|
.unmodifiableList(Arrays.asList(AzureConstants.ACCOUNT_NAME, AzureConstants.ACCOUNT_KEY, AzureConstants.CONTAINER, BLOB));
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
|
||||||
|
return PROPERTIES;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException {
|
||||||
|
FlowFile flowFile = session.get();
|
||||||
|
if (flowFile == null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
final long startNanos = System.nanoTime();
|
||||||
|
|
||||||
|
String containerName = context.getProperty(AzureConstants.CONTAINER).evaluateAttributeExpressions(flowFile).getValue();
|
||||||
|
String blobPath = context.getProperty(BLOB).evaluateAttributeExpressions(flowFile).getValue();
|
||||||
|
|
||||||
|
try {
|
||||||
|
CloudStorageAccount storageAccount = createStorageConnection(context, flowFile);
|
||||||
|
CloudBlobClient blobClient = storageAccount.createCloudBlobClient();
|
||||||
|
CloudBlobContainer container = blobClient.getContainerReference(containerName);
|
||||||
|
|
||||||
|
final Map<String, String> attributes = new HashMap<>();
|
||||||
|
final CloudBlob blob = container.getBlockBlobReference(blobPath);
|
||||||
|
|
||||||
|
// TODO - we may be able do fancier things with ranges and
|
||||||
|
// distribution of download over threads, investigate
|
||||||
|
flowFile = session.write(flowFile, new OutputStreamCallback() {
|
||||||
|
@Override
|
||||||
|
public void process(OutputStream os) throws IOException {
|
||||||
|
try {
|
||||||
|
blob.download(os);
|
||||||
|
} catch (StorageException e) {
|
||||||
|
throw new IOException(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
long length = blob.getProperties().getLength();
|
||||||
|
attributes.put("azure.length", String.valueOf(length));
|
||||||
|
|
||||||
|
if (!attributes.isEmpty()) {
|
||||||
|
flowFile = session.putAllAttributes(flowFile, attributes);
|
||||||
|
}
|
||||||
|
|
||||||
|
session.transfer(flowFile, REL_SUCCESS);
|
||||||
|
final long transferMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos);
|
||||||
|
session.getProvenanceReporter().fetch(flowFile, blob.getSnapshotQualifiedUri().toString(), transferMillis);
|
||||||
|
|
||||||
|
} catch (IllegalArgumentException | URISyntaxException | StorageException e1) {
|
||||||
|
flowFile = session.penalize(flowFile);
|
||||||
|
session.transfer(flowFile, REL_FAILURE);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,193 @@
|
||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
* contributor license agreements. See the NOTICE file distributed with
|
||||||
|
* this work for additional information regarding copyright ownership.
|
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
* (the "License"); you may not use this file except in compliance with
|
||||||
|
* the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.nifi.processors.azure.storage;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.net.URISyntaxException;
|
||||||
|
import java.security.InvalidKeyException;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.Arrays;
|
||||||
|
import java.util.Collections;
|
||||||
|
import java.util.EnumSet;
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
import org.apache.nifi.annotation.behavior.InputRequirement;
|
||||||
|
import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
|
||||||
|
import org.apache.nifi.annotation.behavior.Stateful;
|
||||||
|
import org.apache.nifi.annotation.behavior.TriggerSerially;
|
||||||
|
import org.apache.nifi.annotation.behavior.WritesAttribute;
|
||||||
|
import org.apache.nifi.annotation.behavior.WritesAttributes;
|
||||||
|
import org.apache.nifi.annotation.documentation.CapabilityDescription;
|
||||||
|
import org.apache.nifi.annotation.documentation.SeeAlso;
|
||||||
|
import org.apache.nifi.annotation.documentation.Tags;
|
||||||
|
import org.apache.nifi.components.PropertyDescriptor;
|
||||||
|
import org.apache.nifi.components.state.Scope;
|
||||||
|
import org.apache.nifi.processor.ProcessContext;
|
||||||
|
import org.apache.nifi.processor.util.StandardValidators;
|
||||||
|
import org.apache.nifi.processors.azure.AzureConstants;
|
||||||
|
import org.apache.nifi.processors.azure.storage.utils.BlobInfo;
|
||||||
|
import org.apache.nifi.processors.azure.storage.utils.BlobInfo.Builder;
|
||||||
|
import org.apache.nifi.processors.standard.AbstractListProcessor;
|
||||||
|
|
||||||
|
import com.microsoft.azure.storage.CloudStorageAccount;
|
||||||
|
import com.microsoft.azure.storage.OperationContext;
|
||||||
|
import com.microsoft.azure.storage.StorageException;
|
||||||
|
import com.microsoft.azure.storage.StorageUri;
|
||||||
|
import com.microsoft.azure.storage.blob.BlobListingDetails;
|
||||||
|
import com.microsoft.azure.storage.blob.BlobProperties;
|
||||||
|
import com.microsoft.azure.storage.blob.BlobRequestOptions;
|
||||||
|
import com.microsoft.azure.storage.blob.CloudBlob;
|
||||||
|
import com.microsoft.azure.storage.blob.CloudBlobClient;
|
||||||
|
import com.microsoft.azure.storage.blob.CloudBlobContainer;
|
||||||
|
import com.microsoft.azure.storage.blob.CloudBlockBlob;
|
||||||
|
import com.microsoft.azure.storage.blob.ListBlobItem;
|
||||||
|
|
||||||
|
@TriggerSerially
|
||||||
|
@Tags({ "azure", "microsoft", "cloud", "storage", "blob" })
|
||||||
|
@SeeAlso({ FetchAzureBlobStorage.class })
|
||||||
|
@CapabilityDescription("Lists blobs in an Azure Storage container. Listing details are attached to an empty FlowFile for use with FetchAzureBlobStorage")
|
||||||
|
@InputRequirement(Requirement.INPUT_FORBIDDEN)
|
||||||
|
@WritesAttributes({ @WritesAttribute(attribute = "azure.container", description = "The name of the azure container"),
|
||||||
|
@WritesAttribute(attribute = "azure.blobname", description = "The name of the azure blob"), @WritesAttribute(attribute = "azure.primaryUri", description = "Primary location for blob content"),
|
||||||
|
@WritesAttribute(attribute = "azure.secondaryUri", description = "Secondary location for blob content"), @WritesAttribute(attribute = "azure.etag", description = "Etag for the Azure blob"),
|
||||||
|
@WritesAttribute(attribute = "azure.length", description = "Length of the blob"), @WritesAttribute(attribute = "azure.timestamp", description = "The timestamp in Azure for the blob"),
|
||||||
|
@WritesAttribute(attribute = "mime.type", description = "MimeType of the content"), @WritesAttribute(attribute = "lang", description = "Language code for the content"),
|
||||||
|
@WritesAttribute(attribute = "azure.blobtype", description = "This is the type of blob and can be either page or block type") })
|
||||||
|
@Stateful(scopes = { Scope.LOCAL, Scope.CLUSTER }, description = "After performing a listing of blobs, the timestamp of the newest blob is stored. "
|
||||||
|
+ "This allows the Processor to list only blobs that have been added or modified after " + "this date the next time that the Processor is run.")
|
||||||
|
public class ListAzureBlobStorage extends AbstractListProcessor<BlobInfo> {
|
||||||
|
|
||||||
|
private static final PropertyDescriptor PREFIX = new PropertyDescriptor.Builder().name("Prefix").description("Search prefix for listing").addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
|
||||||
|
.expressionLanguageSupported(true).required(false).build();
|
||||||
|
|
||||||
|
public static final List<PropertyDescriptor> PROPERTIES = Collections.unmodifiableList(Arrays.asList(AzureConstants.ACCOUNT_NAME, AzureConstants.ACCOUNT_KEY, AzureConstants.CONTAINER, PREFIX));
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
|
||||||
|
return PROPERTIES;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected Map<String, String> createAttributes(BlobInfo entity, ProcessContext context) {
|
||||||
|
final Map<String, String> attributes = new HashMap<>();
|
||||||
|
attributes.put("azure.etag", entity.getEtag());
|
||||||
|
attributes.put("azure.primaryUri", entity.getPrimaryUri());
|
||||||
|
attributes.put("azure.secondaryUri", entity.getSecondaryUri());
|
||||||
|
attributes.put("azure.blobname", entity.getName());
|
||||||
|
attributes.put("azure.blobtype", entity.getBlobType());
|
||||||
|
attributes.put("azure.length", String.valueOf(entity.getLength()));
|
||||||
|
attributes.put("azure.timestamp", String.valueOf(entity.getTimestamp()));
|
||||||
|
attributes.put("mime.type", entity.getContentType());
|
||||||
|
attributes.put("lang", entity.getContentLanguage());
|
||||||
|
|
||||||
|
return attributes;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected String getPath(final ProcessContext context) {
|
||||||
|
return context.getProperty(AzureConstants.CONTAINER).evaluateAttributeExpressions().getValue();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected boolean isListingResetNecessary(final PropertyDescriptor property) {
|
||||||
|
// TODO - implement
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected Scope getStateScope(final ProcessContext context) {
|
||||||
|
return Scope.CLUSTER;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected List<BlobInfo> performListing(final ProcessContext context, final Long minTimestamp) throws IOException {
|
||||||
|
String containerName = context.getProperty(AzureConstants.CONTAINER).evaluateAttributeExpressions().getValue();
|
||||||
|
String prefix = context.getProperty(PREFIX).evaluateAttributeExpressions().getValue();
|
||||||
|
if (prefix == null) {
|
||||||
|
prefix = "";
|
||||||
|
}
|
||||||
|
final List<BlobInfo> listing = new ArrayList<>();
|
||||||
|
try {
|
||||||
|
CloudStorageAccount storageAccount = createStorageConnection(context);
|
||||||
|
CloudBlobClient blobClient = storageAccount.createCloudBlobClient();
|
||||||
|
CloudBlobContainer container = blobClient.getContainerReference(containerName);
|
||||||
|
|
||||||
|
BlobRequestOptions blobRequestOptions = null;
|
||||||
|
OperationContext operationContext = null;
|
||||||
|
|
||||||
|
for (ListBlobItem blob : container.listBlobs(prefix, true, EnumSet.of(BlobListingDetails.METADATA), blobRequestOptions, operationContext)) {
|
||||||
|
if (blob instanceof CloudBlob) {
|
||||||
|
CloudBlob cloudBlob = (CloudBlob) blob;
|
||||||
|
BlobProperties properties = cloudBlob.getProperties();
|
||||||
|
StorageUri uri = cloudBlob.getSnapshotQualifiedStorageUri();
|
||||||
|
|
||||||
|
Builder builder = new BlobInfo.Builder().primaryUri(uri.getPrimaryUri().toString()).secondaryUri(uri.getSecondaryUri().toString()).contentType(properties.getContentType())
|
||||||
|
.contentLanguage(properties.getContentLanguage()).etag(properties.getEtag()).lastModifiedTime(properties.getLastModified().getTime()).length(properties.getLength());
|
||||||
|
|
||||||
|
if (blob instanceof CloudBlockBlob) {
|
||||||
|
builder.blobType(AzureConstants.BLOCK);
|
||||||
|
} else {
|
||||||
|
builder.blobType(AzureConstants.PAGE);
|
||||||
|
}
|
||||||
|
listing.add(builder.build());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (IllegalArgumentException | URISyntaxException | StorageException e) {
|
||||||
|
throw (new IOException(e));
|
||||||
|
}
|
||||||
|
return listing;
|
||||||
|
}
|
||||||
|
|
||||||
|
protected static CloudStorageAccount createStorageConnection(ProcessContext context) {
|
||||||
|
final String accountName = context.getProperty(AzureConstants.ACCOUNT_NAME).evaluateAttributeExpressions().getValue();
|
||||||
|
final String accountKey = context.getProperty(AzureConstants.ACCOUNT_KEY).evaluateAttributeExpressions().getValue();
|
||||||
|
final String storageConnectionString = String.format("DefaultEndpointsProtocol=http;AccountName=%s;AccountKey=%s", accountName, accountKey);
|
||||||
|
try {
|
||||||
|
return createStorageAccountFromConnectionString(storageConnectionString);
|
||||||
|
} catch (InvalidKeyException | URISyntaxException e) {
|
||||||
|
throw new IllegalArgumentException(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Validates the connection string and returns the storage account. The connection string must be in the Azure connection string format.
|
||||||
|
*
|
||||||
|
* @param storageConnectionString
|
||||||
|
* Connection string for the storage service or the emulator
|
||||||
|
* @return The newly created CloudStorageAccount object
|
||||||
|
*
|
||||||
|
*/
|
||||||
|
private static CloudStorageAccount createStorageAccountFromConnectionString(String storageConnectionString) throws IllegalArgumentException, URISyntaxException, InvalidKeyException {
|
||||||
|
|
||||||
|
CloudStorageAccount storageAccount;
|
||||||
|
try {
|
||||||
|
storageAccount = CloudStorageAccount.parse(storageConnectionString);
|
||||||
|
} catch (IllegalArgumentException | URISyntaxException e) {
|
||||||
|
System.out.println("\nConnection string specifies an invalid URI.");
|
||||||
|
System.out.println("Please confirm the connection string is in the Azure connection string format.");
|
||||||
|
throw e;
|
||||||
|
} catch (InvalidKeyException e) {
|
||||||
|
System.out.println("\nConnection string specifies an invalid key.");
|
||||||
|
System.out.println("Please confirm the AccountName and AccountKey in the connection string are valid.");
|
||||||
|
throw e;
|
||||||
|
}
|
||||||
|
return storageAccount;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,116 @@
|
||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
* contributor license agreements. See the NOTICE file distributed with
|
||||||
|
* this work for additional information regarding copyright ownership.
|
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
* (the "License"); you may not use this file except in compliance with
|
||||||
|
* the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.nifi.processors.azure.storage;
|
||||||
|
|
||||||
|
import java.io.BufferedInputStream;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.io.InputStream;
|
||||||
|
import java.net.URISyntaxException;
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.concurrent.TimeUnit;
|
||||||
|
|
||||||
|
import org.apache.nifi.annotation.behavior.InputRequirement;
|
||||||
|
import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
|
||||||
|
import org.apache.nifi.annotation.behavior.WritesAttribute;
|
||||||
|
import org.apache.nifi.annotation.behavior.WritesAttributes;
|
||||||
|
import org.apache.nifi.annotation.documentation.CapabilityDescription;
|
||||||
|
import org.apache.nifi.annotation.documentation.SeeAlso;
|
||||||
|
import org.apache.nifi.annotation.documentation.Tags;
|
||||||
|
import org.apache.nifi.flowfile.FlowFile;
|
||||||
|
import org.apache.nifi.processor.ProcessContext;
|
||||||
|
import org.apache.nifi.processor.ProcessSession;
|
||||||
|
import org.apache.nifi.processor.exception.ProcessException;
|
||||||
|
import org.apache.nifi.processor.io.InputStreamCallback;
|
||||||
|
import org.apache.nifi.processors.azure.AbstractAzureBlobProcessor;
|
||||||
|
import org.apache.nifi.processors.azure.AzureConstants;
|
||||||
|
|
||||||
|
import com.microsoft.azure.storage.CloudStorageAccount;
|
||||||
|
import com.microsoft.azure.storage.StorageException;
|
||||||
|
import com.microsoft.azure.storage.blob.BlobProperties;
|
||||||
|
import com.microsoft.azure.storage.blob.CloudBlob;
|
||||||
|
import com.microsoft.azure.storage.blob.CloudBlobClient;
|
||||||
|
import com.microsoft.azure.storage.blob.CloudBlobContainer;
|
||||||
|
|
||||||
|
@Tags({ "azure", "microsoft", "cloud", "storage", "blob" })
|
||||||
|
@SeeAlso({ ListAzureBlobStorage.class, FetchAzureBlobStorage.class })
|
||||||
|
@CapabilityDescription("Puts content into an Azure Storage Blob")
|
||||||
|
@InputRequirement(Requirement.INPUT_REQUIRED)
|
||||||
|
@WritesAttributes({ @WritesAttribute(attribute = "azure.container", description = "The name of the azure container"),
|
||||||
|
@WritesAttribute(attribute = "azure.blobname", description = "The name of the azure blob"),
|
||||||
|
@WritesAttribute(attribute = "azure.primaryUri", description = "Primary location for blob content"),
|
||||||
|
@WritesAttribute(attribute = "azure.etag", description = "Etag for the Azure blob"),
|
||||||
|
@WritesAttribute(attribute = "azure.length", description = "Length of the blob"),
|
||||||
|
@WritesAttribute(attribute = "azure.timestamp", description = "The timestamp in Azure for the blob"),
|
||||||
|
@WritesAttribute(attribute = "azure.blobtype", description = "This is the type of blob and can be either page or block type") })
|
||||||
|
public class PutAzureBlobStorage extends AbstractAzureBlobProcessor {
|
||||||
|
|
||||||
|
public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
|
||||||
|
FlowFile flowFile = session.get();
|
||||||
|
if (flowFile == null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
final long startNanos = System.nanoTime();
|
||||||
|
|
||||||
|
String containerName = context.getProperty(AzureConstants.CONTAINER).evaluateAttributeExpressions(flowFile).getValue();
|
||||||
|
|
||||||
|
String blobPath = context.getProperty(BLOB).evaluateAttributeExpressions(flowFile).getValue();
|
||||||
|
|
||||||
|
try {
|
||||||
|
CloudStorageAccount storageAccount = createStorageConnection(context, flowFile);
|
||||||
|
CloudBlobClient blobClient = storageAccount.createCloudBlobClient();
|
||||||
|
CloudBlobContainer container = blobClient.getContainerReference(containerName);
|
||||||
|
|
||||||
|
CloudBlob blob = container.getBlockBlobReference(blobPath);
|
||||||
|
|
||||||
|
final Map<String, String> attributes = new HashMap<>();
|
||||||
|
long length = flowFile.getSize();
|
||||||
|
session.read(flowFile, new InputStreamCallback() {
|
||||||
|
@Override
|
||||||
|
public void process(final InputStream rawIn) throws IOException {
|
||||||
|
final InputStream in = new BufferedInputStream(rawIn);
|
||||||
|
try {
|
||||||
|
blob.upload(in, length);
|
||||||
|
BlobProperties properties = blob.getProperties();
|
||||||
|
attributes.put("azure.container", containerName);
|
||||||
|
attributes.put("azure.primaryUri", blob.getSnapshotQualifiedUri().toString());
|
||||||
|
attributes.put("azure.etag", properties.getEtag());
|
||||||
|
attributes.put("azure.length", String.valueOf(length));
|
||||||
|
attributes.put("azure.timestamp", String.valueOf(properties.getLastModified()));
|
||||||
|
} catch (StorageException | URISyntaxException e) {
|
||||||
|
throw new IOException(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!attributes.isEmpty()) {
|
||||||
|
flowFile = session.putAllAttributes(flowFile, attributes);
|
||||||
|
}
|
||||||
|
session.transfer(flowFile, REL_SUCCESS);
|
||||||
|
|
||||||
|
final long transferMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos);
|
||||||
|
session.getProvenanceReporter().send(flowFile, blob.getSnapshotQualifiedUri().toString(), transferMillis);
|
||||||
|
|
||||||
|
} catch (IllegalArgumentException | URISyntaxException | StorageException e) {
|
||||||
|
getLogger().error("Failed to put Azure blob {}", new Object[]{blobPath}, e);
|
||||||
|
flowFile = session.penalize(flowFile);
|
||||||
|
session.transfer(flowFile, REL_FAILURE);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,188 @@
|
||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
* contributor license agreements. See the NOTICE file distributed with
|
||||||
|
* this work for additional information regarding copyright ownership.
|
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
* (the "License"); you may not use this file except in compliance with
|
||||||
|
* the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.nifi.processors.azure.storage.utils;
|
||||||
|
|
||||||
|
import java.io.Serializable;
|
||||||
|
|
||||||
|
import org.apache.nifi.processors.standard.util.ListableEntity;
|
||||||
|
|
||||||
|
public class BlobInfo implements Comparable<BlobInfo>, Serializable, ListableEntity {
|
||||||
|
private static final long serialVersionUID = 1L;
|
||||||
|
|
||||||
|
private final String primaryUri;
|
||||||
|
private final String secondaryUri;
|
||||||
|
private final String contentType;
|
||||||
|
private final String contentLanguage;
|
||||||
|
private final String etag;
|
||||||
|
private final long lastModifiedTime;
|
||||||
|
private final long length;
|
||||||
|
private final String blobType;
|
||||||
|
|
||||||
|
public static long getSerialversionuid() {
|
||||||
|
return serialVersionUID;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getPrimaryUri() {
|
||||||
|
return primaryUri;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getSecondaryUri() {
|
||||||
|
return secondaryUri;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getContentType() {
|
||||||
|
return contentType;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getContentLanguage() {
|
||||||
|
return contentLanguage;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getEtag() {
|
||||||
|
return etag;
|
||||||
|
}
|
||||||
|
|
||||||
|
public long getLastModifiedTime() {
|
||||||
|
return lastModifiedTime;
|
||||||
|
}
|
||||||
|
|
||||||
|
public long getLength() {
|
||||||
|
return length;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getBlobType() {
|
||||||
|
return blobType;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static final class Builder {
|
||||||
|
private String primaryUri;
|
||||||
|
private String secondaryUri;
|
||||||
|
private String contentType;
|
||||||
|
private String contentLanguage;
|
||||||
|
private String etag;
|
||||||
|
private long lastModifiedTime;
|
||||||
|
private long length;
|
||||||
|
private String blobType;
|
||||||
|
|
||||||
|
public Builder primaryUri(String primaryUri) {
|
||||||
|
this.primaryUri = primaryUri;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Builder secondaryUri(String secondaryUri) {
|
||||||
|
this.secondaryUri = secondaryUri;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Builder contentType(String contentType) {
|
||||||
|
this.contentType = contentType;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Builder contentLanguage(String contentLanguage) {
|
||||||
|
this.contentLanguage = contentLanguage;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Builder etag(String etag) {
|
||||||
|
this.etag = etag;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Builder lastModifiedTime(long lastModifiedTime) {
|
||||||
|
this.lastModifiedTime = lastModifiedTime;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Builder length(long length) {
|
||||||
|
this.length = length;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Builder blobType(String blobType) {
|
||||||
|
this.blobType = blobType;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
public BlobInfo build() {
|
||||||
|
return new BlobInfo(this);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public int hashCode() {
|
||||||
|
final int prime = 31;
|
||||||
|
int result = 1;
|
||||||
|
result = prime * result + ((etag == null) ? 0 : etag.hashCode());
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean equals(Object obj) {
|
||||||
|
if (this == obj) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
if (obj == null) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (getClass() != obj.getClass()) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
BlobInfo other = (BlobInfo) obj;
|
||||||
|
if (etag == null) {
|
||||||
|
if (other.etag != null) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
} else if (!etag.equals(other.etag)) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public int compareTo(BlobInfo o) {
|
||||||
|
return etag.compareTo(o.etag);
|
||||||
|
}
|
||||||
|
|
||||||
|
protected BlobInfo(final Builder builder) {
|
||||||
|
this.primaryUri = builder.primaryUri;
|
||||||
|
this.secondaryUri = builder.secondaryUri;
|
||||||
|
this.contentType = builder.contentType;
|
||||||
|
this.contentLanguage = builder.contentLanguage;
|
||||||
|
this.etag = builder.etag;
|
||||||
|
this.lastModifiedTime = builder.lastModifiedTime;
|
||||||
|
this.length = builder.length;
|
||||||
|
this.blobType = builder.blobType;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String getName() {
|
||||||
|
String primaryUri = getPrimaryUri();
|
||||||
|
return primaryUri.substring(primaryUri.lastIndexOf('/') + 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String getIdentifier() {
|
||||||
|
return getPrimaryUri();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public long getTimestamp() {
|
||||||
|
return getLastModifiedTime();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -14,3 +14,6 @@
|
||||||
# limitations under the License.
|
# limitations under the License.
|
||||||
org.apache.nifi.processors.azure.eventhub.PutAzureEventHub
|
org.apache.nifi.processors.azure.eventhub.PutAzureEventHub
|
||||||
org.apache.nifi.processors.azure.eventhub.GetAzureEventHub
|
org.apache.nifi.processors.azure.eventhub.GetAzureEventHub
|
||||||
|
org.apache.nifi.processors.azure.storage.FetchAzureBlobStorage
|
||||||
|
org.apache.nifi.processors.azure.storage.ListAzureBlobStorage
|
||||||
|
org.apache.nifi.processors.azure.storage.PutAzureBlobStorage
|
||||||
|
|
@ -0,0 +1,106 @@
|
||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
* contributor license agreements. See the NOTICE file distributed with
|
||||||
|
* this work for additional information regarding copyright ownership.
|
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
* (the "License"); you may not use this file except in compliance with
|
||||||
|
* the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.nifi.processors.azure.storage;
|
||||||
|
|
||||||
|
import static org.junit.Assert.fail;
|
||||||
|
|
||||||
|
import java.io.FileInputStream;
|
||||||
|
import java.io.FileNotFoundException;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.net.URISyntaxException;
|
||||||
|
import java.security.InvalidKeyException;
|
||||||
|
import java.util.Properties;
|
||||||
|
|
||||||
|
import org.apache.nifi.util.file.FileUtils;
|
||||||
|
import org.junit.AfterClass;
|
||||||
|
import org.junit.BeforeClass;
|
||||||
|
|
||||||
|
import com.microsoft.azure.storage.CloudStorageAccount;
|
||||||
|
import com.microsoft.azure.storage.StorageException;
|
||||||
|
import com.microsoft.azure.storage.blob.CloudBlob;
|
||||||
|
import com.microsoft.azure.storage.blob.CloudBlobClient;
|
||||||
|
import com.microsoft.azure.storage.blob.CloudBlobContainer;
|
||||||
|
import com.microsoft.azure.storage.blob.DeleteSnapshotsOption;
|
||||||
|
import com.microsoft.azure.storage.blob.ListBlobItem;
|
||||||
|
import com.microsoft.azure.storage.table.CloudTable;
|
||||||
|
import com.microsoft.azure.storage.table.CloudTableClient;
|
||||||
|
|
||||||
|
public abstract class AbstractAzureIT {
|
||||||
|
protected static final String CREDENTIALS_FILE = System.getProperty("user.home") + "/azure-credentials.PROPERTIES";
|
||||||
|
public static final String TEST_CONTAINER_NAME = "nifitest";
|
||||||
|
|
||||||
|
private static final Properties CONFIG;
|
||||||
|
protected static final String TEST_BLOB_NAME = "testing";
|
||||||
|
protected static final String TEST_TABLE_NAME = "testing";
|
||||||
|
|
||||||
|
static {
|
||||||
|
final FileInputStream fis;
|
||||||
|
CONFIG = new Properties();
|
||||||
|
try {
|
||||||
|
fis = new FileInputStream(CREDENTIALS_FILE);
|
||||||
|
try {
|
||||||
|
CONFIG.load(fis);
|
||||||
|
} catch (IOException e) {
|
||||||
|
fail("Could not open credentials file " + CREDENTIALS_FILE + ": " + e.getLocalizedMessage());
|
||||||
|
} finally {
|
||||||
|
FileUtils.closeQuietly(fis);
|
||||||
|
}
|
||||||
|
} catch (FileNotFoundException e) {
|
||||||
|
fail("Could not open credentials file " + CREDENTIALS_FILE + ": " + e.getLocalizedMessage());
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
@BeforeClass
|
||||||
|
public static void oneTimeSetup() throws StorageException, InvalidKeyException, URISyntaxException {
|
||||||
|
CloudBlobContainer container = getContainer();
|
||||||
|
container.createIfNotExists();
|
||||||
|
}
|
||||||
|
|
||||||
|
@AfterClass
|
||||||
|
public static void tearDown() throws InvalidKeyException, URISyntaxException, StorageException {
|
||||||
|
CloudBlobContainer container = getContainer();
|
||||||
|
for (ListBlobItem blob : container.listBlobs()) {
|
||||||
|
if (blob instanceof CloudBlob) {
|
||||||
|
((CloudBlob) blob).delete(DeleteSnapshotsOption.INCLUDE_SNAPSHOTS, null, null, null);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String getAccountName() {
|
||||||
|
return CONFIG.getProperty("accountName");
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String getAccountKey() {
|
||||||
|
return CONFIG.getProperty("accountKey");
|
||||||
|
}
|
||||||
|
|
||||||
|
protected static CloudBlobContainer getContainer() throws InvalidKeyException, URISyntaxException, StorageException {
|
||||||
|
String storageConnectionString = String.format("DefaultEndpointsProtocol=http;AccountName=%s;AccountKey=%s", getAccountName(), getAccountKey());
|
||||||
|
CloudStorageAccount storageAccount = CloudStorageAccount.parse(storageConnectionString);
|
||||||
|
CloudBlobClient blobClient = storageAccount.createCloudBlobClient();
|
||||||
|
return blobClient.getContainerReference(TEST_CONTAINER_NAME);
|
||||||
|
}
|
||||||
|
|
||||||
|
protected static CloudTable getTable() throws InvalidKeyException, URISyntaxException, StorageException {
|
||||||
|
String storageConnectionString = String.format("DefaultEndpointsProtocol=http;AccountName=%s;AccountKey=%s", getAccountName(), getAccountKey());
|
||||||
|
CloudStorageAccount storageAccount = CloudStorageAccount.parse(storageConnectionString);
|
||||||
|
CloudTableClient tableClient = storageAccount.createCloudTableClient();
|
||||||
|
return tableClient.getTableReference(TEST_TABLE_NAME);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,61 @@
|
||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
* contributor license agreements. See the NOTICE file distributed with
|
||||||
|
* this work for additional information regarding copyright ownership.
|
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
* (the "License"); you may not use this file except in compliance with
|
||||||
|
* the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.nifi.processors.azure.storage;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.net.URISyntaxException;
|
||||||
|
import java.security.InvalidKeyException;
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
import org.apache.nifi.processors.azure.AzureConstants;
|
||||||
|
import org.apache.nifi.util.MockFlowFile;
|
||||||
|
import org.apache.nifi.util.TestRunner;
|
||||||
|
import org.apache.nifi.util.TestRunners;
|
||||||
|
import org.junit.Test;
|
||||||
|
|
||||||
|
import com.microsoft.azure.storage.StorageException;
|
||||||
|
|
||||||
|
public class ITFetchAzureBlobStorage extends AbstractAzureIT {
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testFetchingBlob() throws InvalidKeyException, URISyntaxException, StorageException, IOException {
|
||||||
|
final TestRunner runner = TestRunners.newTestRunner(new FetchAzureBlobStorage());
|
||||||
|
|
||||||
|
runner.setValidateExpressionUsage(true);
|
||||||
|
|
||||||
|
runner.setProperty(AzureConstants.ACCOUNT_NAME, getAccountName());
|
||||||
|
runner.setProperty(AzureConstants.ACCOUNT_KEY, getAccountKey());
|
||||||
|
runner.setProperty(AzureConstants.CONTAINER, TEST_CONTAINER_NAME);
|
||||||
|
runner.setProperty(FetchAzureBlobStorage.BLOB, "${azure.blobname}");
|
||||||
|
|
||||||
|
final Map<String, String> attributes = new HashMap<>();
|
||||||
|
attributes.put("azure.primaryUri", "http://" + getAccountName() + ".blob.core.windows.net/" + TEST_CONTAINER_NAME + "/" + TEST_BLOB_NAME);
|
||||||
|
attributes.put("azure.blobname", TEST_BLOB_NAME);
|
||||||
|
attributes.put("azure.blobtype", AzureConstants.BLOCK);
|
||||||
|
runner.enqueue(new byte[0], attributes);
|
||||||
|
runner.run();
|
||||||
|
|
||||||
|
runner.assertAllFlowFilesTransferred(FetchAzureBlobStorage.REL_SUCCESS, 1);
|
||||||
|
List<MockFlowFile> flowFilesForRelationship = runner.getFlowFilesForRelationship(FetchAzureBlobStorage.REL_SUCCESS);
|
||||||
|
for (MockFlowFile flowFile : flowFilesForRelationship) {
|
||||||
|
flowFile.assertContentEquals("0123456789".getBytes());
|
||||||
|
flowFile.assertAttributeEquals("azure.length", "10");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,75 @@
|
||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
* contributor license agreements. See the NOTICE file distributed with
|
||||||
|
* this work for additional information regarding copyright ownership.
|
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
* (the "License"); you may not use this file except in compliance with
|
||||||
|
* the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.nifi.processors.azure.storage;
|
||||||
|
|
||||||
|
import java.io.ByteArrayInputStream;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.io.InputStream;
|
||||||
|
import java.net.URISyntaxException;
|
||||||
|
import java.security.InvalidKeyException;
|
||||||
|
|
||||||
|
import org.apache.nifi.processors.azure.AzureConstants;
|
||||||
|
import org.apache.nifi.util.MockFlowFile;
|
||||||
|
import org.apache.nifi.util.TestRunner;
|
||||||
|
import org.apache.nifi.util.TestRunners;
|
||||||
|
import org.junit.AfterClass;
|
||||||
|
import org.junit.BeforeClass;
|
||||||
|
import org.junit.Test;
|
||||||
|
|
||||||
|
import com.microsoft.azure.storage.StorageException;
|
||||||
|
import com.microsoft.azure.storage.blob.CloudBlob;
|
||||||
|
import com.microsoft.azure.storage.blob.CloudBlobContainer;
|
||||||
|
|
||||||
|
public class ITListAzureBlobStorage extends AbstractAzureIT {
|
||||||
|
|
||||||
|
@BeforeClass
|
||||||
|
public static void setupSomeFiles() throws InvalidKeyException, URISyntaxException, StorageException, IOException {
|
||||||
|
CloudBlobContainer container = getContainer();
|
||||||
|
container.createIfNotExists();
|
||||||
|
|
||||||
|
CloudBlob blob = container.getBlockBlobReference(TEST_BLOB_NAME);
|
||||||
|
byte[] buf = "0123456789".getBytes();
|
||||||
|
InputStream in = new ByteArrayInputStream(buf);
|
||||||
|
blob.upload(in, 10);
|
||||||
|
}
|
||||||
|
|
||||||
|
@AfterClass
|
||||||
|
public static void tearDown() throws InvalidKeyException, URISyntaxException, StorageException {
|
||||||
|
CloudBlobContainer container = getContainer();
|
||||||
|
container.deleteIfExists();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testListsAzureBlobStorageContent() {
|
||||||
|
final TestRunner runner = TestRunners.newTestRunner(new ListAzureBlobStorage());
|
||||||
|
|
||||||
|
runner.setProperty(AzureConstants.ACCOUNT_NAME, getAccountName());
|
||||||
|
runner.setProperty(AzureConstants.ACCOUNT_KEY, getAccountKey());
|
||||||
|
runner.setProperty(AzureConstants.CONTAINER, TEST_CONTAINER_NAME);
|
||||||
|
|
||||||
|
// requires multiple runs to deal with List processor checking
|
||||||
|
runner.run(3);
|
||||||
|
|
||||||
|
runner.assertTransferCount(ListAzureBlobStorage.REL_SUCCESS, 1);
|
||||||
|
runner.assertAllFlowFilesTransferred(ListAzureBlobStorage.REL_SUCCESS, 1);
|
||||||
|
|
||||||
|
for (MockFlowFile entry : runner.getFlowFilesForRelationship(ListAzureBlobStorage.REL_SUCCESS)) {
|
||||||
|
entry.assertAttributeEquals("azure.length", "10");
|
||||||
|
entry.assertAttributeEquals("mime.type", "application/octet-stream");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,51 @@
|
||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
* contributor license agreements. See the NOTICE file distributed with
|
||||||
|
* this work for additional information regarding copyright ownership.
|
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
* (the "License"); you may not use this file except in compliance with
|
||||||
|
* the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.nifi.processors.azure.storage;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
import org.apache.nifi.processors.azure.AzureConstants;
|
||||||
|
import org.apache.nifi.util.MockFlowFile;
|
||||||
|
import org.apache.nifi.util.TestRunner;
|
||||||
|
import org.apache.nifi.util.TestRunners;
|
||||||
|
import org.junit.Test;
|
||||||
|
|
||||||
|
public class ITPutAzureStorageBlob extends AbstractAzureIT {
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testPuttingBlob() throws IOException {
|
||||||
|
final TestRunner runner = TestRunners.newTestRunner(new PutAzureBlobStorage());
|
||||||
|
|
||||||
|
runner.setValidateExpressionUsage(true);
|
||||||
|
|
||||||
|
runner.setProperty(AzureConstants.ACCOUNT_NAME, getAccountName());
|
||||||
|
runner.setProperty(AzureConstants.ACCOUNT_KEY, getAccountKey());
|
||||||
|
runner.setProperty(AzureConstants.CONTAINER, TEST_CONTAINER_NAME);
|
||||||
|
runner.setProperty(FetchAzureBlobStorage.BLOB, "testingUpload");
|
||||||
|
|
||||||
|
runner.enqueue("0123456789".getBytes());
|
||||||
|
runner.run();
|
||||||
|
|
||||||
|
runner.assertAllFlowFilesTransferred(PutAzureBlobStorage.REL_SUCCESS, 1);
|
||||||
|
List<MockFlowFile> flowFilesForRelationship = runner.getFlowFilesForRelationship(PutAzureBlobStorage.REL_SUCCESS);
|
||||||
|
for (MockFlowFile flowFile : flowFilesForRelationship) {
|
||||||
|
flowFile.assertContentEquals("0123456789".getBytes());
|
||||||
|
flowFile.assertAttributeEquals("azure.length", "10");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
Loading…
Reference in New Issue