From 942030579a055b568eef9de2849e804ad034f174 Mon Sep 17 00:00:00 2001 From: sjyang18 Date: Wed, 20 May 2020 02:47:14 +0000 Subject: [PATCH] NIFI-7386 Added Azure Azurite emulator credentials service This closes #4286 Signed-off-by: Joey Frazee --- .../queue/AbstractAzureQueueStorage.java | 9 +-- .../storage/utils/AzureStorageUtils.java | 29 +++++-- ...eEmulatorCredentialsControllerService.java | 79 +++++++++++++++++++ ...zureStorageEmulatorCredentialsDetails.java | 28 +++++++ ...g.apache.nifi.controller.ControllerService | 1 + ...eEmulatorCredentialsControllerService.java | 55 +++++++++++++ .../AzureStorageCredentialsDetails.java | 4 + 7 files changed, 191 insertions(+), 14 deletions(-) create mode 100644 nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/services/azure/storage/AzureStorageEmulatorCredentialsControllerService.java create mode 100644 nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/services/azure/storage/AzureStorageEmulatorCredentialsDetails.java create mode 100644 nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/test/java/org/apache/nifi/services/azure/storage/TestAzureStorageEmulatorCredentialsControllerService.java diff --git a/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/processors/azure/storage/queue/AbstractAzureQueueStorage.java b/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/processors/azure/storage/queue/AbstractAzureQueueStorage.java index c077df6cc2..968cdc2aaa 100644 --- a/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/processors/azure/storage/queue/AbstractAzureQueueStorage.java +++ b/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/processors/azure/storage/queue/AbstractAzureQueueStorage.java @@ -68,15 +68,8 @@ public abstract class AbstractAzureQueueStorage extends AbstractProcessor { protected final CloudQueueClient createCloudQueueClient(final ProcessContext context, final FlowFile flowFile) throws URISyntaxException { final AzureStorageCredentialsDetails storageCredentialsDetails = AzureStorageUtils.getStorageCredentialsDetails(context, flowFile); - final CloudStorageAccount cloudStorageAccount = - new CloudStorageAccount( - storageCredentialsDetails.getStorageCredentials(), - true, - storageCredentialsDetails.getStorageSuffix(), - storageCredentialsDetails.getStorageAccountName() - ); + final CloudStorageAccount cloudStorageAccount = AzureStorageUtils.getCloudStorageAccount(storageCredentialsDetails); final CloudQueueClient cloudQueueClient = cloudStorageAccount.createCloudQueueClient(); - return cloudQueueClient; } diff --git a/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/processors/azure/storage/utils/AzureStorageUtils.java b/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/processors/azure/storage/utils/AzureStorageUtils.java index 8eebb2c1c8..cf3c8a00ae 100644 --- a/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/processors/azure/storage/utils/AzureStorageUtils.java +++ b/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/processors/azure/storage/utils/AzureStorageUtils.java @@ -16,6 +16,7 @@ */ package org.apache.nifi.processors.azure.storage.utils; +import java.net.URI; import java.net.URISyntaxException; import java.util.ArrayList; import java.util.Collection; @@ -44,6 +45,7 @@ import org.apache.nifi.proxy.ProxyConfiguration; import org.apache.nifi.proxy.ProxySpec; import org.apache.nifi.services.azure.storage.AzureStorageCredentialsDetails; import org.apache.nifi.services.azure.storage.AzureStorageCredentialsService; +import org.apache.nifi.services.azure.storage.AzureStorageEmulatorCredentialsDetails; public final class AzureStorageUtils { public static final String BLOCK = "Block"; @@ -152,16 +154,31 @@ public final class AzureStorageUtils { */ public static CloudBlobClient createCloudBlobClient(ProcessContext context, ComponentLog logger, FlowFile flowFile) throws URISyntaxException { final AzureStorageCredentialsDetails storageCredentialsDetails = getStorageCredentialsDetails(context, flowFile); - final CloudStorageAccount cloudStorageAccount = new CloudStorageAccount( - storageCredentialsDetails.getStorageCredentials(), - true, - storageCredentialsDetails.getStorageSuffix(), - storageCredentialsDetails.getStorageAccountName()); + final CloudStorageAccount cloudStorageAccount = getCloudStorageAccount(storageCredentialsDetails); final CloudBlobClient cloudBlobClient = cloudStorageAccount.createCloudBlobClient(); - return cloudBlobClient; } + public static CloudStorageAccount getCloudStorageAccount(final AzureStorageCredentialsDetails storageCredentialsDetails) throws URISyntaxException { + final CloudStorageAccount cloudStorageAccount; + if (storageCredentialsDetails instanceof AzureStorageEmulatorCredentialsDetails) { + AzureStorageEmulatorCredentialsDetails emulatorCredentials = (AzureStorageEmulatorCredentialsDetails) storageCredentialsDetails; + final String proxyUri = emulatorCredentials.getDevelopmentStorageProxyUri(); + if (proxyUri != null) { + cloudStorageAccount = CloudStorageAccount.getDevelopmentStorageAccount(new URI(proxyUri)); + } else { + cloudStorageAccount = CloudStorageAccount.getDevelopmentStorageAccount(); + } + } else { + cloudStorageAccount = new CloudStorageAccount( + storageCredentialsDetails.getStorageCredentials(), + true, + storageCredentialsDetails.getStorageSuffix(), + storageCredentialsDetails.getStorageAccountName()); + } + return cloudStorageAccount; + } + public static AzureStorageCredentialsDetails getStorageCredentialsDetails(PropertyContext context, FlowFile flowFile) { final Map attributes = flowFile != null ? flowFile.getAttributes() : Collections.emptyMap(); diff --git a/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/services/azure/storage/AzureStorageEmulatorCredentialsControllerService.java b/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/services/azure/storage/AzureStorageEmulatorCredentialsControllerService.java new file mode 100644 index 0000000000..42aac0eced --- /dev/null +++ b/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/services/azure/storage/AzureStorageEmulatorCredentialsControllerService.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.services.azure.storage; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import org.apache.nifi.annotation.documentation.CapabilityDescription; +import org.apache.nifi.annotation.documentation.Tags; +import org.apache.nifi.annotation.lifecycle.OnEnabled; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.components.ValidationContext; +import org.apache.nifi.components.ValidationResult; +import org.apache.nifi.controller.AbstractControllerService; +import org.apache.nifi.controller.ConfigurationContext; +import org.apache.nifi.processor.util.StandardValidators; + +@Tags({ "azure", "microsoft", "emulator", "storage", "blob", "queue", "credentials" }) +@CapabilityDescription("Defines credentials for Azure Storage processors that connects to Azurite emulator.") +public class AzureStorageEmulatorCredentialsControllerService extends AbstractControllerService implements AzureStorageCredentialsService { + + public static final PropertyDescriptor DEVELOPMENT_STORAGE_PROXY_URI = new PropertyDescriptor.Builder() + .name("azurite-uri") + .displayName("Storage Emulator URI") + .description("URI to connect to Azure Storage Emulator (Azurite)") + .required(false) + .sensitive(false) + .addValidator(StandardValidators.URI_VALIDATOR) + .build(); + + private static final List PROPERTIES = + Collections.unmodifiableList(Arrays.asList(DEVELOPMENT_STORAGE_PROXY_URI)); + + private String azuriteProxyUri; + + @Override + protected List getSupportedPropertyDescriptors() { + return PROPERTIES; + } + + @Override + protected Collection customValidate(final ValidationContext validationContext) { + final List results = new ArrayList<>(); + return results; + } + + @OnEnabled + public void onEnabled(final ConfigurationContext context) { + this.azuriteProxyUri = context.getProperty(DEVELOPMENT_STORAGE_PROXY_URI).getValue(); + } + + public String getProxyUri() { + return azuriteProxyUri; + } + + @Override + public AzureStorageCredentialsDetails getStorageCredentialsDetails(final Map attributes) { + return new AzureStorageEmulatorCredentialsDetails(azuriteProxyUri); + + } +} diff --git a/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/services/azure/storage/AzureStorageEmulatorCredentialsDetails.java b/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/services/azure/storage/AzureStorageEmulatorCredentialsDetails.java new file mode 100644 index 0000000000..1f0f79803e --- /dev/null +++ b/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/services/azure/storage/AzureStorageEmulatorCredentialsDetails.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.services.azure.storage; + +public class AzureStorageEmulatorCredentialsDetails extends AzureStorageCredentialsDetails { + private String developmentStorageProxyUri; + + public AzureStorageEmulatorCredentialsDetails(String developmentStorageProxyUri) { + this.developmentStorageProxyUri = developmentStorageProxyUri; + } + public String getDevelopmentStorageProxyUri() { + return developmentStorageProxyUri; + } +} diff --git a/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/resources/META-INF/services/org.apache.nifi.controller.ControllerService b/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/resources/META-INF/services/org.apache.nifi.controller.ControllerService index 5e63d4b190..3d5ffffdfa 100644 --- a/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/resources/META-INF/services/org.apache.nifi.controller.ControllerService +++ b/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/resources/META-INF/services/org.apache.nifi.controller.ControllerService @@ -16,3 +16,4 @@ org.apache.nifi.services.azure.storage.AzureStorageCredentialsControllerService org.apache.nifi.services.azure.storage.AzureStorageCredentialsControllerServiceLookup org.apache.nifi.services.azure.storage.ADLSCredentialsControllerService org.apache.nifi.services.azure.cosmos.document.AzureCosmosDBClientService +org.apache.nifi.services.azure.storage.AzureStorageEmulatorCredentialsControllerService diff --git a/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/test/java/org/apache/nifi/services/azure/storage/TestAzureStorageEmulatorCredentialsControllerService.java b/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/test/java/org/apache/nifi/services/azure/storage/TestAzureStorageEmulatorCredentialsControllerService.java new file mode 100644 index 0000000000..40f5545d05 --- /dev/null +++ b/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/test/java/org/apache/nifi/services/azure/storage/TestAzureStorageEmulatorCredentialsControllerService.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.services.azure.storage; + +import org.apache.nifi.reporting.InitializationException; +import org.apache.nifi.util.NoOpProcessor; +import org.apache.nifi.util.TestRunner; +import org.apache.nifi.util.TestRunners; +import org.junit.Before; +import org.junit.Test; + +public class TestAzureStorageEmulatorCredentialsControllerService { + + private static final String TEST_ENVIRONMENT_URI = "http://127.0.0.1"; + + private TestRunner runner; + private AzureStorageCredentialsService credentialsService; + + @Before + public void setUp() throws InitializationException { + runner = TestRunners.newTestRunner(NoOpProcessor.class); + credentialsService = new AzureStorageEmulatorCredentialsControllerService(); + runner.addControllerService("credentials-service", credentialsService); + } + + @Test + public void testValidWithProxyURI() { + configureProxyURI(); + + runner.assertValid(credentialsService); + } + + @Test + public void testValidWithoutProxyURI() { + runner.assertValid(credentialsService); + } + + private void configureProxyURI() { + runner.setProperty(credentialsService, AzureStorageEmulatorCredentialsControllerService.DEVELOPMENT_STORAGE_PROXY_URI, TEST_ENVIRONMENT_URI); + } +} diff --git a/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-services-api/src/main/java/org/apache/nifi/services/azure/storage/AzureStorageCredentialsDetails.java b/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-services-api/src/main/java/org/apache/nifi/services/azure/storage/AzureStorageCredentialsDetails.java index fc1ccaf7e4..f009ab35fc 100644 --- a/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-services-api/src/main/java/org/apache/nifi/services/azure/storage/AzureStorageCredentialsDetails.java +++ b/nifi-nar-bundles/nifi-azure-bundle/nifi-azure-services-api/src/main/java/org/apache/nifi/services/azure/storage/AzureStorageCredentialsDetails.java @@ -26,6 +26,10 @@ public class AzureStorageCredentialsDetails { private final StorageCredentials storageCredentials; + public AzureStorageCredentialsDetails() { + this(null, null, null); + } + @Deprecated public AzureStorageCredentialsDetails(String storageAccountName, StorageCredentials storageCredentials) { this(storageAccountName, null, storageCredentials);