From f742a3a6accd7695a32b1e4c42c2ec93619a7b9a Mon Sep 17 00:00:00 2001 From: zenfenan Date: Sat, 5 May 2018 14:45:40 +0530 Subject: [PATCH] NIFI-5156: Updated GCP SDK to latest version Signed-off-by: Pierre Villard This closes #2680. --- .../nifi-gcp-processors/pom.xml | 16 +------------ .../processors/gcp/AbstractGCPProcessor.java | 5 ++-- .../gcp/storage/AbstractGCSProcessor.java | 24 +++++++------------ .../processors/gcp/storage/ListGCSBucket.java | 2 +- .../MockCredentialsFactoryProcessor.java | 6 ++--- .../MockCredentialsServiceProcessor.java | 6 ++--- .../processors/gcp/storage/AbstractGCSIT.java | 6 ++--- .../gcp/storage/AbstractGCSTest.java | 5 +--- .../gcp/storage/FetchGCSObjectTest.java | 5 ---- .../gcp/storage/ListGCSBucketTest.java | 3 ++- nifi-nar-bundles/nifi-gcp-bundle/pom.xml | 16 +++++++++++++ 11 files changed, 40 insertions(+), 54 deletions(-) diff --git a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/pom.xml b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/pom.xml index c9119b8f59..eeb757b700 100644 --- a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/pom.xml +++ b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/pom.xml @@ -63,21 +63,12 @@ com.google.cloud - google-cloud - 0.8.0 + google-cloud-storage - - org.json - json - com.google.code.findbugs jsr305 - - commons-fileupload - commons-fileupload - @@ -85,11 +76,6 @@ json 1.8 - - commons-fileupload - commons-fileupload - 1.3.3 - diff --git a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/AbstractGCPProcessor.java b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/AbstractGCPProcessor.java index 5f51194ecb..0505d63745 100644 --- a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/AbstractGCPProcessor.java +++ b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/AbstractGCPProcessor.java @@ -17,8 +17,8 @@ package org.apache.nifi.processors.gcp; import com.google.auth.oauth2.GoogleCredentials; -import com.google.cloud.HttpServiceOptions; import com.google.cloud.Service; +import com.google.cloud.ServiceOptions; import com.google.common.collect.ImmutableList; import org.apache.nifi.annotation.lifecycle.OnScheduled; import org.apache.nifi.components.PropertyDescriptor; @@ -36,8 +36,7 @@ import java.util.List; */ public abstract class AbstractGCPProcessor< CloudService extends Service, - CloudServiceRpc, - CloudServiceOptions extends HttpServiceOptions> extends AbstractProcessor { + CloudServiceOptions extends ServiceOptions> extends AbstractProcessor { public static final PropertyDescriptor PROJECT_ID = new PropertyDescriptor .Builder().name("gcp-project-id") diff --git a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/storage/AbstractGCSProcessor.java b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/storage/AbstractGCSProcessor.java index 7cc854d02b..bed596bc3b 100644 --- a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/storage/AbstractGCSProcessor.java +++ b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/storage/AbstractGCSProcessor.java @@ -18,12 +18,12 @@ package org.apache.nifi.processors.gcp.storage; import com.google.api.client.http.HttpTransport; import com.google.api.client.http.javanet.NetHttpTransport; +import com.google.api.gax.retrying.RetrySettings; import com.google.auth.http.HttpTransportFactory; import com.google.auth.oauth2.GoogleCredentials; -import com.google.cloud.RetryParams; +import com.google.cloud.http.HttpTransportOptions; import com.google.cloud.storage.Storage; import com.google.cloud.storage.StorageOptions; -import com.google.cloud.storage.spi.StorageRpc; import com.google.common.collect.ImmutableList; import org.apache.nifi.components.PropertyDescriptor; import org.apache.nifi.processor.ProcessContext; @@ -44,7 +44,7 @@ import java.util.Set; * * Every GCS processor operation requires a bucket, whether it's reading or writing from said bucket. */ -public abstract class AbstractGCSProcessor extends AbstractGCPProcessor { +public abstract class AbstractGCSProcessor extends AbstractGCPProcessor { public static final Relationship REL_SUCCESS = new Relationship.Builder().name("success") .description("FlowFiles are routed to this relationship after a successful Google Cloud Storage operation.") @@ -80,25 +80,19 @@ public abstract class AbstractGCSProcessor extends AbstractGCPProcessor 0) { - storageOptionsBuilder.setHttpTransportFactory(new HttpTransportFactory() { + storageOptionsBuilder.setTransportOptions(HttpTransportOptions.newBuilder().setHttpTransportFactory(new HttpTransportFactory() { @Override public HttpTransport create() { - final HttpTransport transport = new NetHttpTransport.Builder() - .setProxy( - new Proxy( - Proxy.Type.HTTP, new InetSocketAddress(proxyHost, proxyPort) - ) - ) + return new NetHttpTransport.Builder() + .setProxy(new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyHost, proxyPort))) .build(); - return transport; } - }); + }).build()); } return storageOptionsBuilder.build(); } diff --git a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/storage/ListGCSBucket.java b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/storage/ListGCSBucket.java index 6bb3812353..46b48fb0ae 100644 --- a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/storage/ListGCSBucket.java +++ b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/main/java/org/apache/nifi/processors/gcp/storage/ListGCSBucket.java @@ -16,7 +16,7 @@ */ package org.apache.nifi.processors.gcp.storage; -import com.google.cloud.Page; +import com.google.api.gax.paging.Page; import com.google.cloud.storage.Acl; import com.google.cloud.storage.Blob; import com.google.cloud.storage.BlobInfo; diff --git a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/credentials/factory/MockCredentialsFactoryProcessor.java b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/credentials/factory/MockCredentialsFactoryProcessor.java index 0734c400e3..84614e961e 100644 --- a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/credentials/factory/MockCredentialsFactoryProcessor.java +++ b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/credentials/factory/MockCredentialsFactoryProcessor.java @@ -17,8 +17,8 @@ package org.apache.nifi.processors.gcp.credentials.factory; import com.google.auth.oauth2.GoogleCredentials; -import com.google.cloud.HttpServiceOptions; import com.google.cloud.Service; +import com.google.cloud.ServiceOptions; import org.apache.nifi.components.PropertyDescriptor; import org.apache.nifi.components.ValidationContext; import org.apache.nifi.components.ValidationResult; @@ -53,8 +53,8 @@ public class MockCredentialsFactoryProcessor extends AbstractGCPProcessor { } @Override - protected HttpServiceOptions getServiceOptions(ProcessContext context, GoogleCredentials credentials) { - HttpServiceOptions mockOptions = mock(HttpServiceOptions.class); + protected ServiceOptions getServiceOptions(ProcessContext context, GoogleCredentials credentials) { + ServiceOptions mockOptions = mock(ServiceOptions.class); Service mockService = mock(Service.class); when(mockOptions.getService()).thenReturn(mockService); diff --git a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/credentials/service/MockCredentialsServiceProcessor.java b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/credentials/service/MockCredentialsServiceProcessor.java index 36c71d6104..e351e4efbc 100644 --- a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/credentials/service/MockCredentialsServiceProcessor.java +++ b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/credentials/service/MockCredentialsServiceProcessor.java @@ -17,8 +17,8 @@ package org.apache.nifi.processors.gcp.credentials.service; import com.google.auth.oauth2.GoogleCredentials; -import com.google.cloud.HttpServiceOptions; import com.google.cloud.Service; +import com.google.cloud.ServiceOptions; import org.apache.nifi.components.PropertyDescriptor; import org.apache.nifi.processor.ProcessContext; import org.apache.nifi.processor.ProcessSession; @@ -43,8 +43,8 @@ public class MockCredentialsServiceProcessor extends AbstractGCPProcessor { } @Override - protected HttpServiceOptions getServiceOptions(ProcessContext context, GoogleCredentials credentials) { - HttpServiceOptions mockOptions = mock(HttpServiceOptions.class); + protected ServiceOptions getServiceOptions(ProcessContext context, GoogleCredentials credentials) { + ServiceOptions mockOptions = mock(ServiceOptions.class); Service mockService = mock(Service.class); when(mockOptions.getService()).thenReturn(mockService); diff --git a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/storage/AbstractGCSIT.java b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/storage/AbstractGCSIT.java index 725f824d0b..d66df784e3 100644 --- a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/storage/AbstractGCSIT.java +++ b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/storage/AbstractGCSIT.java @@ -33,7 +33,6 @@ import org.junit.BeforeClass; import org.junit.experimental.categories.Category; import java.util.Arrays; -import java.util.Iterator; import static org.junit.Assert.fail; @@ -78,10 +77,9 @@ public abstract class AbstractGCSIT { public static void tearDown() { try { // Empty the bucket before deleting it. - Iterator blobIterator = storage.list(BUCKET, Storage.BlobListOption.versions(true)).iterateAll(); + Iterable blobIterable = storage.list(BUCKET, Storage.BlobListOption.versions(true)).iterateAll(); - while(blobIterator.hasNext()) { - Blob blob = blobIterator.next(); + for (final Blob blob : blobIterable) { storage.delete(blob.getBlobId()); } diff --git a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/storage/AbstractGCSTest.java b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/storage/AbstractGCSTest.java index 6b35cde490..8b94c47c9e 100644 --- a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/storage/AbstractGCSTest.java +++ b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/storage/AbstractGCSTest.java @@ -88,10 +88,7 @@ public abstract class AbstractGCSTest { PROJECT_ID, options.getProjectId()); assertEquals("Retry counts should match", - RETRIES.intValue(), options.getRetryParams().getRetryMinAttempts()); - - assertEquals("Retry counts should match", - RETRIES.intValue(), options.getRetryParams().getRetryMaxAttempts()); + RETRIES.intValue(), options.getRetrySettings().getMaxAttempts()); assertSame("Credentials should be configured correctly", mockCredentials, options.getCredentials()); diff --git a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/storage/FetchGCSObjectTest.java b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/storage/FetchGCSObjectTest.java index 0d8b20952d..5e51270556 100644 --- a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/storage/FetchGCSObjectTest.java +++ b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/storage/FetchGCSObjectTest.java @@ -141,11 +141,6 @@ public class FetchGCSObjectTest extends AbstractGCSTest { } - @Override - public void chunkSize(int i) { - - } - @Override public void setChunkSize(int i) { diff --git a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/storage/ListGCSBucketTest.java b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/storage/ListGCSBucketTest.java index 6c21d913fb..eca81bbaa2 100644 --- a/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/storage/ListGCSBucketTest.java +++ b/nifi-nar-bundles/nifi-gcp-bundle/nifi-gcp-processors/src/test/java/org/apache/nifi/processors/gcp/storage/ListGCSBucketTest.java @@ -16,7 +16,8 @@ */ package org.apache.nifi.processors.gcp.storage; -import com.google.cloud.Page; + +import com.google.api.gax.paging.Page; import com.google.cloud.storage.Acl; import com.google.cloud.storage.Blob; import com.google.cloud.storage.BlobInfo; diff --git a/nifi-nar-bundles/nifi-gcp-bundle/pom.xml b/nifi-nar-bundles/nifi-gcp-bundle/pom.xml index 36a33be279..341633ccf4 100644 --- a/nifi-nar-bundles/nifi-gcp-bundle/pom.xml +++ b/nifi-nar-bundles/nifi-gcp-bundle/pom.xml @@ -26,6 +26,22 @@ 1.7.0-SNAPSHOT pom + + 0.47.0-alpha + + + + + + com.google.cloud + google-cloud + ${google.cloud.sdk.version} + pom + import + + + + nifi-gcp-services-api nifi-gcp-services-api-nar