From c073527b1bbd622411a3b00d81c280f66d00bf1a Mon Sep 17 00:00:00 2001 From: Timea Barna Date: Thu, 1 Aug 2024 10:11:15 +0200 Subject: [PATCH] Revert "NIFI-13439 Add performance tracking to ProcessGroupStatus" This reverts commit efe421527f7f3add32df7cdc3f1b36becf88dbd1. --- .../controller/status/ProcessGroupStatus.java | 28 -- .../status/ProcessingPerformanceStatus.java | 107 ----- .../controller/status/ProcessorStatus.java | 12 - .../status/ProcessGroupStatusSnapshotDTO.java | 13 - .../ProcessingPerformanceStatusDTO.java | 100 ---- .../status/ProcessorStatusSnapshotDTO.java | 13 - .../ProcessingPerformanceStatusMerger.java | 34 -- .../nifi/cluster/manager/StatusMerger.java | 7 - .../nifi/reporting/AbstractEventAccess.java | 29 -- .../reporting/PerformanceMetricsUtil.java | 37 -- .../apache/nifi/web/api/dto/DtoFactory.java | 26 -- .../nifi-grpc-bundle/nifi-grpc-common/pom.xml | 148 ++++++ .../grpc/FlowFileIngestService.java | 163 +++++++ .../FlowFileIngestServiceInterceptor.java | 161 +++++++ .../processors/grpc/GRPCAttributeNames.java | 23 + .../grpc/ssl/SslContextProvider.java | 62 +++ .../grpc/util/BackpressureChecker.java | 47 ++ .../resources/proto/flowfile_service.proto | 59 +++ .../grpc/ssl/SslContextProviderTest.java | 109 +++++ .../nifi-grpc-bundle/nifi-grpc-nar/pom.xml | 41 ++ .../nifi-grpc-nar/src/main/resources/LICENSE | 259 +++++++++++ .../nifi-grpc-nar/src/main/resources/NOTICE | 295 ++++++++++++ .../nifi-grpc-processors/pom.xml | 46 ++ .../nifi/processors/grpc/InvokeGRPC.java | 438 ++++++++++++++++++ .../nifi/processors/grpc/ListenGRPC.java | 233 ++++++++++ .../org.apache.nifi.processor.Processor | 16 + .../nifi/processors/grpc/ITListenGRPC.java | 170 +++++++ .../nifi/processors/grpc/TestGRPCClient.java | 132 ++++++ .../nifi/processors/grpc/TestGRPCServer.java | 166 +++++++ .../nifi/processors/grpc/TestInvokeGRPC.java | 422 +++++++++++++++++ .../nifi/processors/grpc/TestListenGRPC.java | 131 ++++++ nifi-nar-bundles/nifi-grpc-bundle/pom.xml | 49 ++ .../ProcessGroupStatusEnumerator.java | 7 +- .../ProcessGroupStatusTable.java | 12 +- .../ProcessorStatusEnumerator.java | 7 +- .../processorstatus/ProcessorStatusTable.java | 14 +- .../sql/TestQueryNiFiReportingTask.java | 2 +- 37 files changed, 3176 insertions(+), 442 deletions(-) delete mode 100644 nifi-api/src/main/java/org/apache/nifi/controller/status/ProcessingPerformanceStatus.java delete mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/status/ProcessingPerformanceStatusDTO.java delete mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/manager/ProcessingPerformanceStatusMerger.java delete mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/reporting/PerformanceMetricsUtil.java create mode 100644 nifi-nar-bundles/nifi-grpc-bundle/nifi-grpc-common/pom.xml create mode 100644 nifi-nar-bundles/nifi-grpc-bundle/nifi-grpc-common/src/main/java/org/apache/nifi/processors/grpc/FlowFileIngestService.java create mode 100644 nifi-nar-bundles/nifi-grpc-bundle/nifi-grpc-common/src/main/java/org/apache/nifi/processors/grpc/FlowFileIngestServiceInterceptor.java create mode 100644 nifi-nar-bundles/nifi-grpc-bundle/nifi-grpc-common/src/main/java/org/apache/nifi/processors/grpc/GRPCAttributeNames.java create mode 100644 nifi-nar-bundles/nifi-grpc-bundle/nifi-grpc-common/src/main/java/org/apache/nifi/processors/grpc/ssl/SslContextProvider.java create mode 100644 nifi-nar-bundles/nifi-grpc-bundle/nifi-grpc-common/src/main/java/org/apache/nifi/processors/grpc/util/BackpressureChecker.java create mode 100644 nifi-nar-bundles/nifi-grpc-bundle/nifi-grpc-common/src/main/resources/proto/flowfile_service.proto create mode 100644 nifi-nar-bundles/nifi-grpc-bundle/nifi-grpc-common/src/test/java/org/apache/nifi/processors/grpc/ssl/SslContextProviderTest.java create mode 100644 nifi-nar-bundles/nifi-grpc-bundle/nifi-grpc-nar/pom.xml create mode 100644 nifi-nar-bundles/nifi-grpc-bundle/nifi-grpc-nar/src/main/resources/LICENSE create mode 100644 nifi-nar-bundles/nifi-grpc-bundle/nifi-grpc-nar/src/main/resources/NOTICE create mode 100644 nifi-nar-bundles/nifi-grpc-bundle/nifi-grpc-processors/pom.xml create mode 100644 nifi-nar-bundles/nifi-grpc-bundle/nifi-grpc-processors/src/main/java/org/apache/nifi/processors/grpc/InvokeGRPC.java create mode 100644 nifi-nar-bundles/nifi-grpc-bundle/nifi-grpc-processors/src/main/java/org/apache/nifi/processors/grpc/ListenGRPC.java create mode 100644 nifi-nar-bundles/nifi-grpc-bundle/nifi-grpc-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor create mode 100644 nifi-nar-bundles/nifi-grpc-bundle/nifi-grpc-processors/src/test/java/org/apache/nifi/processors/grpc/ITListenGRPC.java create mode 100644 nifi-nar-bundles/nifi-grpc-bundle/nifi-grpc-processors/src/test/java/org/apache/nifi/processors/grpc/TestGRPCClient.java create mode 100644 nifi-nar-bundles/nifi-grpc-bundle/nifi-grpc-processors/src/test/java/org/apache/nifi/processors/grpc/TestGRPCServer.java create mode 100644 nifi-nar-bundles/nifi-grpc-bundle/nifi-grpc-processors/src/test/java/org/apache/nifi/processors/grpc/TestInvokeGRPC.java create mode 100644 nifi-nar-bundles/nifi-grpc-bundle/nifi-grpc-processors/src/test/java/org/apache/nifi/processors/grpc/TestListenGRPC.java create mode 100644 nifi-nar-bundles/nifi-grpc-bundle/pom.xml diff --git a/nifi-api/src/main/java/org/apache/nifi/controller/status/ProcessGroupStatus.java b/nifi-api/src/main/java/org/apache/nifi/controller/status/ProcessGroupStatus.java index 31c77028c6..cf3ba5459c 100644 --- a/nifi-api/src/main/java/org/apache/nifi/controller/status/ProcessGroupStatus.java +++ b/nifi-api/src/main/java/org/apache/nifi/controller/status/ProcessGroupStatus.java @@ -56,8 +56,6 @@ public class ProcessGroupStatus implements Cloneable { private Collection inputPortStatus = new ArrayList<>(); private Collection outputPortStatus = new ArrayList<>(); - private ProcessingPerformanceStatus processingPerformanceStatus; - public String getId() { return id; } @@ -266,14 +264,6 @@ public class ProcessGroupStatus implements Cloneable { this.processingNanos = processingNanos; } - public ProcessingPerformanceStatus getProcessingPerformanceStatus() { - return processingPerformanceStatus; - } - - public void setProcessingPerformanceStatus(ProcessingPerformanceStatus processingPerformanceStatus) { - this.processingPerformanceStatus = processingPerformanceStatus; - } - @Override public ProcessGroupStatus clone() { final ProcessGroupStatus clonedObj = new ProcessGroupStatus(); @@ -297,7 +287,6 @@ public class ProcessGroupStatus implements Cloneable { clonedObj.flowFilesTransferred = flowFilesTransferred; clonedObj.bytesTransferred = bytesTransferred; clonedObj.processingNanos = processingNanos; - clonedObj.processingPerformanceStatus = processingPerformanceStatus; if (connectionStatus != null) { final Collection statusList = new ArrayList<>(); @@ -420,9 +409,6 @@ public class ProcessGroupStatus implements Cloneable { builder.append(status); } - builder.append(", processingPerformanceStatus="); - builder.append(processingPerformanceStatus); - builder.append("]"); return builder.toString(); } @@ -624,20 +610,6 @@ public class ProcessGroupStatus implements Cloneable { } target.setRemoteProcessGroupStatus(mergedRemoteGroupMap.values()); - - final ProcessingPerformanceStatus targetPerformanceStatus = target.getProcessingPerformanceStatus(); - final ProcessingPerformanceStatus toMergePerformanceStatus = toMerge.getProcessingPerformanceStatus(); - - if (targetPerformanceStatus != null && toMergePerformanceStatus != null) { - targetPerformanceStatus.setIdentifier(toMergePerformanceStatus.getIdentifier()); - targetPerformanceStatus.setCpuDuration(targetPerformanceStatus.getCpuDuration() + toMergePerformanceStatus.getCpuDuration()); - targetPerformanceStatus.setContentReadDuration(targetPerformanceStatus.getContentReadDuration() + toMergePerformanceStatus.getContentReadDuration()); - targetPerformanceStatus.setContentWriteDuration(targetPerformanceStatus.getContentWriteDuration() + toMergePerformanceStatus.getContentWriteDuration()); - targetPerformanceStatus.setSessionCommitDuration(targetPerformanceStatus.getSessionCommitDuration() + toMergePerformanceStatus.getSessionCommitDuration()); - targetPerformanceStatus.setGarbageCollectionDuration(targetPerformanceStatus.getGarbageCollectionDuration() + toMergePerformanceStatus.getGarbageCollectionDuration()); - } else { - target.setProcessingPerformanceStatus(targetPerformanceStatus); - } } public static FlowFileAvailability mergeFlowFileAvailability(final FlowFileAvailability availabilityA, final FlowFileAvailability availabilityB) { diff --git a/nifi-api/src/main/java/org/apache/nifi/controller/status/ProcessingPerformanceStatus.java b/nifi-api/src/main/java/org/apache/nifi/controller/status/ProcessingPerformanceStatus.java deleted file mode 100644 index e87894c343..0000000000 --- a/nifi-api/src/main/java/org/apache/nifi/controller/status/ProcessingPerformanceStatus.java +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.controller.status; - -public class ProcessingPerformanceStatus implements Cloneable { - - private String identifier; - private long cpuDuration; - private long contentReadDuration; - private long contentWriteDuration; - private long sessionCommitDuration; - private long garbageCollectionDuration; - - public String getIdentifier() { - return identifier; - } - - public void setIdentifier(String identifier) { - this.identifier = identifier; - } - - public long getCpuDuration() { - return cpuDuration; - } - - public void setCpuDuration(long cpuDuration) { - this.cpuDuration = cpuDuration; - } - - public long getContentReadDuration() { - return contentReadDuration; - } - - public void setContentReadDuration(long contentReadDuration) { - this.contentReadDuration = contentReadDuration; - } - - public long getContentWriteDuration() { - return contentWriteDuration; - } - - public void setContentWriteDuration(long contentWriteDuration) { - this.contentWriteDuration = contentWriteDuration; - } - - public long getSessionCommitDuration() { - return sessionCommitDuration; - } - - public void setSessionCommitDuration(long sessionCommitDuration) { - this.sessionCommitDuration = sessionCommitDuration; - } - - public long getGarbageCollectionDuration() { - return garbageCollectionDuration; - } - - public void setGarbageCollectionDuration(long garbageCollectionDuration) { - this.garbageCollectionDuration = garbageCollectionDuration; - } - - @Override - public ProcessingPerformanceStatus clone() { - final ProcessingPerformanceStatus clonedObj = new ProcessingPerformanceStatus(); - - clonedObj.identifier = identifier; - clonedObj.cpuDuration = cpuDuration; - clonedObj.contentReadDuration = contentReadDuration; - clonedObj.contentWriteDuration = contentWriteDuration; - clonedObj.sessionCommitDuration = sessionCommitDuration; - clonedObj.garbageCollectionDuration = garbageCollectionDuration; - return clonedObj; - } - - @Override - public String toString() { - StringBuilder builder = new StringBuilder(); - builder.append("ProcessorPerformanceStatus [Group ID= "); - builder.append(identifier); - builder.append(", cpuDuration= "); - builder.append(cpuDuration); - builder.append(", contentReadDuration= "); - builder.append(contentReadDuration); - builder.append(", contentWriteDuration= "); - builder.append(contentWriteDuration); - builder.append(", sessionCommitDuration= "); - builder.append(sessionCommitDuration); - builder.append(", garbageCollectionDuration= "); - builder.append(garbageCollectionDuration); - builder.append("]"); - return builder.toString(); - } -} diff --git a/nifi-api/src/main/java/org/apache/nifi/controller/status/ProcessorStatus.java b/nifi-api/src/main/java/org/apache/nifi/controller/status/ProcessorStatus.java index 380829d38f..ba90534239 100644 --- a/nifi-api/src/main/java/org/apache/nifi/controller/status/ProcessorStatus.java +++ b/nifi-api/src/main/java/org/apache/nifi/controller/status/ProcessorStatus.java @@ -48,7 +48,6 @@ public class ProcessorStatus implements Cloneable { private int flowFilesSent; private long bytesSent; private Map counters; - private ProcessingPerformanceStatus processingPerformanceStatus; public String getId() { return id; @@ -242,14 +241,6 @@ public class ProcessorStatus implements Cloneable { this.counters = counters; } - public ProcessingPerformanceStatus getProcessingPerformanceStatus() { - return processingPerformanceStatus; - } - - public void setProcessingPerformanceStatus(ProcessingPerformanceStatus processingPerformanceStatus) { - this.processingPerformanceStatus = processingPerformanceStatus; - } - @Override public ProcessorStatus clone() { final ProcessorStatus clonedObj = new ProcessorStatus(); @@ -276,7 +267,6 @@ public class ProcessorStatus implements Cloneable { clonedObj.executionNode = executionNode; clonedObj.type = type; clonedObj.counters = counters == null ? null : new HashMap<>(counters); - clonedObj.processingPerformanceStatus = processingPerformanceStatus; return clonedObj; } @@ -317,8 +307,6 @@ public class ProcessorStatus implements Cloneable { builder.append(terminatedThreadCount); builder.append(", counters="); builder.append(counters); - builder.append(", processingPerformanceStatus="); - builder.append(processingPerformanceStatus); builder.append("]"); return builder.toString(); } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/status/ProcessGroupStatusSnapshotDTO.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/status/ProcessGroupStatusSnapshotDTO.java index a7223a3b06..9047697502 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/status/ProcessGroupStatusSnapshotDTO.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/status/ProcessGroupStatusSnapshotDTO.java @@ -80,9 +80,6 @@ public class ProcessGroupStatusSnapshotDTO implements Cloneable { private Integer terminatedThreadCount = 0; private Long processingNanos = 0L; - @ApiModelProperty("Represents the processing performance for all the processors in the given process group.") - private ProcessingPerformanceStatusDTO processingPerformanceStatus; - /** * The id for the process group. * @@ -510,14 +507,6 @@ public class ProcessGroupStatusSnapshotDTO implements Cloneable { this.processingNanos = processingNanos; } - public ProcessingPerformanceStatusDTO getProcessingPerformanceStatus() { - return processingPerformanceStatus; - } - - public void setProcessingPerformanceStatus(ProcessingPerformanceStatusDTO processingPerformanceStatus) { - this.processingPerformanceStatus = processingPerformanceStatus; - } - @Override public ProcessGroupStatusSnapshotDTO clone() { final ProcessGroupStatusSnapshotDTO other = new ProcessGroupStatusSnapshotDTO(); @@ -595,8 +584,6 @@ public class ProcessGroupStatusSnapshotDTO implements Cloneable { other.setProcessGroupStatusSnapshots(childGroups); } - other.setProcessingPerformanceStatus(getProcessingPerformanceStatus()); - return other; } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/status/ProcessingPerformanceStatusDTO.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/status/ProcessingPerformanceStatusDTO.java deleted file mode 100644 index 6f009171bb..0000000000 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/status/ProcessingPerformanceStatusDTO.java +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.web.api.dto.status; - -import io.swagger.annotations.ApiModelProperty; -import javax.xml.bind.annotation.XmlType; - -/** - * DTO for serializing the performance status of a processor. - */ -@XmlType(name = "processingPerformanceStatus") -public class ProcessingPerformanceStatusDTO implements Cloneable { - private String identifier; - private long cpuDuration; - private long contentReadDuration; - private long contentWriteDuration; - private long sessionCommitDuration; - private long garbageCollectionDuration; - - @ApiModelProperty("The unique ID of the process group that the Processor belongs to") - public String getIdentifier() { - return identifier; - } - - public void setIdentifier(String identifier) { - this.identifier = identifier; - } - - @ApiModelProperty("The number of nanoseconds has spent on CPU usage in the last 5 minutes.") - public long getCpuDuration() { - return cpuDuration; - } - - public void setCpuDuration(long cpuDuration) { - this.cpuDuration = cpuDuration; - } - - @ApiModelProperty("The number of nanoseconds has spent to read content in the last 5 minutes.") - public long getContentReadDuration() { - return contentReadDuration; - } - - public void setContentReadDuration(long contentReadDuration) { - this.contentReadDuration = contentReadDuration; - } - - @ApiModelProperty("The number of nanoseconds has spent to write content in the last 5 minutes.") - public long getContentWriteDuration() { - return contentWriteDuration; - } - - public void setContentWriteDuration(long contentWriteDuration) { - this.contentWriteDuration = contentWriteDuration; - } - - @ApiModelProperty("The number of nanoseconds has spent running to commit sessions the last 5 minutes.") - public long getSessionCommitDuration() { - return sessionCommitDuration; - } - - public void setSessionCommitDuration(long sessionCommitDuration) { - this.sessionCommitDuration = sessionCommitDuration; - } - - @ApiModelProperty("The number of nanoseconds has spent running garbage collection in the last 5 minutes.") - public long getGarbageCollectionDuration() { - return garbageCollectionDuration; - } - - public void setGarbageCollectionDuration(long garbageCollectionDuration) { - this.garbageCollectionDuration = garbageCollectionDuration; - } - - @Override - public ProcessingPerformanceStatusDTO clone() { - final ProcessingPerformanceStatusDTO clonedObj = new ProcessingPerformanceStatusDTO(); - - clonedObj.identifier = identifier; - clonedObj.cpuDuration = cpuDuration; - clonedObj.contentReadDuration = contentReadDuration; - clonedObj.contentWriteDuration = contentWriteDuration; - clonedObj.sessionCommitDuration = sessionCommitDuration; - clonedObj.garbageCollectionDuration = garbageCollectionDuration; - return clonedObj; - } -} \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/status/ProcessorStatusSnapshotDTO.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/status/ProcessorStatusSnapshotDTO.java index 01126f46f3..10c740880c 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/status/ProcessorStatusSnapshotDTO.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/status/ProcessorStatusSnapshotDTO.java @@ -53,9 +53,6 @@ public class ProcessorStatusSnapshotDTO implements Cloneable { private Integer activeThreadCount = 0; private Integer terminatedThreadCount = 0; - @ApiModelProperty("Represents the processor's processing performance.") - private ProcessingPerformanceStatusDTO processingPerformanceStatus; - /* getters / setters */ /** * @return The processor id @@ -300,14 +297,6 @@ public class ProcessorStatusSnapshotDTO implements Cloneable { this.tasksDurationNanos = taskNanos; } - public ProcessingPerformanceStatusDTO getProcessingPerformanceStatus() { - return processingPerformanceStatus; - } - - public void setProcessingPerformanceStatus(ProcessingPerformanceStatusDTO processingPerformanceStatus) { - this.processingPerformanceStatus = processingPerformanceStatus; - } - @Override public ProcessorStatusSnapshotDTO clone() { final ProcessorStatusSnapshotDTO other = new ProcessorStatusSnapshotDTO(); @@ -335,8 +324,6 @@ public class ProcessorStatusSnapshotDTO implements Cloneable { other.setWritten(getWritten()); other.setTasks(getTasks()); - other.setProcessingPerformanceStatus(getProcessingPerformanceStatus()); - return other; } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/manager/ProcessingPerformanceStatusMerger.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/manager/ProcessingPerformanceStatusMerger.java deleted file mode 100644 index 30b6c34f27..0000000000 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/manager/ProcessingPerformanceStatusMerger.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.cluster.manager; - -import org.apache.nifi.web.api.dto.status.ProcessingPerformanceStatusDTO; - -public class ProcessingPerformanceStatusMerger { - public static void mergeStatus(final ProcessingPerformanceStatusDTO target, final ProcessingPerformanceStatusDTO toMerge) { - if (target == null || toMerge == null) { - return; - } - - target.setIdentifier(toMerge.getIdentifier()); - target.setCpuDuration(target.getCpuDuration() + toMerge.getCpuDuration()); - target.setContentReadDuration(target.getContentReadDuration() + toMerge.getContentReadDuration()); - target.setContentWriteDuration(target.getContentWriteDuration() + toMerge.getContentWriteDuration()); - target.setSessionCommitDuration(target.getSessionCommitDuration() + toMerge.getSessionCommitDuration()); - target.setGarbageCollectionDuration(target.getGarbageCollectionDuration() + toMerge.getGarbageCollectionDuration()); - } -} \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/manager/StatusMerger.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/manager/StatusMerger.java index 3ed76db87e..d91c8f1ce4 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/manager/StatusMerger.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/manager/StatusMerger.java @@ -174,9 +174,6 @@ public class StatusMerger { target.setTerminatedThreadCount(target.getTerminatedThreadCount() + toMerge.getTerminatedThreadCount()); target.setProcessingNanos(target.getProcessingNanos() + toMerge.getProcessingNanos()); - - ProcessingPerformanceStatusMerger.mergeStatus(target.getProcessingPerformanceStatus(), toMerge.getProcessingPerformanceStatus()); - updatePrettyPrintedFields(target); // connection status @@ -454,10 +451,6 @@ public class StatusMerger { target.setTasksDurationNanos(target.getTasksDurationNanos() + toMerge.getTasksDurationNanos()); target.setActiveThreadCount(target.getActiveThreadCount() + toMerge.getActiveThreadCount()); target.setTerminatedThreadCount(target.getTerminatedThreadCount() + toMerge.getTerminatedThreadCount()); - - - ProcessingPerformanceStatusMerger.mergeStatus(target.getProcessingPerformanceStatus(), toMerge.getProcessingPerformanceStatus()); - updatePrettyPrintedFields(target); } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/reporting/AbstractEventAccess.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/reporting/AbstractEventAccess.java index c9a597e4eb..132e3fe3f5 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/reporting/AbstractEventAccess.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/reporting/AbstractEventAccess.java @@ -40,7 +40,6 @@ import org.apache.nifi.controller.status.ConnectionStatus; import org.apache.nifi.controller.status.LoadBalanceStatus; import org.apache.nifi.controller.status.PortStatus; import org.apache.nifi.controller.status.ProcessGroupStatus; -import org.apache.nifi.controller.status.ProcessingPerformanceStatus; import org.apache.nifi.controller.status.ProcessorStatus; import org.apache.nifi.controller.status.RemoteProcessGroupStatus; import org.apache.nifi.controller.status.RunStatus; @@ -159,8 +158,6 @@ public abstract class AbstractEventAccess implements EventAccess { int flowFilesTransferred = 0; long bytesTransferred = 0; long processingNanos = 0; - final ProcessingPerformanceStatus performanceStatus = new ProcessingPerformanceStatus(); - performanceStatus.setIdentifier(group.getIdentifier()); final boolean populateChildStatuses = currentDepth <= recursiveStatusDepth; @@ -183,16 +180,6 @@ public abstract class AbstractEventAccess implements EventAccess { bytesSent += procStat.getBytesSent(); processingNanos += procStat.getProcessingNanos(); - - final ProcessingPerformanceStatus processorPerformanceStatus = procStat.getProcessingPerformanceStatus(); - - if (processorPerformanceStatus != null) { - performanceStatus.setCpuDuration(performanceStatus.getCpuDuration() + processorPerformanceStatus.getCpuDuration()); - performanceStatus.setContentReadDuration(performanceStatus.getContentReadDuration() + processorPerformanceStatus.getContentReadDuration()); - performanceStatus.setContentWriteDuration(performanceStatus.getContentWriteDuration() + processorPerformanceStatus.getContentWriteDuration()); - performanceStatus.setSessionCommitDuration(performanceStatus.getSessionCommitDuration() + processorPerformanceStatus.getSessionCommitDuration()); - performanceStatus.setGarbageCollectionDuration(performanceStatus.getGarbageCollectionDuration() + processorPerformanceStatus.getGarbageCollectionDuration()); - } } // set status for local child groups @@ -227,16 +214,6 @@ public abstract class AbstractEventAccess implements EventAccess { bytesTransferred += childGroupStatus.getBytesTransferred(); processingNanos += childGroupStatus.getProcessingNanos(); - - final ProcessingPerformanceStatus childGroupPerformanceStatus = childGroupStatus.getProcessingPerformanceStatus(); - - if (childGroupPerformanceStatus != null) { - performanceStatus.setCpuDuration(performanceStatus.getCpuDuration() + childGroupPerformanceStatus.getCpuDuration()); - performanceStatus.setContentReadDuration(performanceStatus.getContentReadDuration() + childGroupPerformanceStatus.getContentReadDuration()); - performanceStatus.setContentWriteDuration(performanceStatus.getContentWriteDuration() + childGroupPerformanceStatus.getContentWriteDuration()); - performanceStatus.setSessionCommitDuration(performanceStatus.getSessionCommitDuration() + childGroupPerformanceStatus.getSessionCommitDuration()); - performanceStatus.setGarbageCollectionDuration(performanceStatus.getGarbageCollectionDuration() + childGroupPerformanceStatus.getGarbageCollectionDuration()); - } } // set status for remote child groups @@ -516,8 +493,6 @@ public abstract class AbstractEventAccess implements EventAccess { status.setBytesTransferred(bytesTransferred); status.setProcessingNanos(processingNanos); - status.setProcessingPerformanceStatus(performanceStatus); - final VersionControlInformation vci = group.getVersionControlInformation(); if (vci != null) { try { @@ -664,10 +639,6 @@ public abstract class AbstractEventAccess implements EventAccess { if (isProcessorAuthorized) { status.setCounters(flowFileEvent.getCounters()); } - - - final ProcessingPerformanceStatus performanceStatus = PerformanceMetricsUtil.getPerformanceMetrics(flowFileEvent, procNode); - status.setProcessingPerformanceStatus(performanceStatus); } // Determine the run status and get any validation error... only validating while STOPPED diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/reporting/PerformanceMetricsUtil.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/reporting/PerformanceMetricsUtil.java deleted file mode 100644 index 6877e4d3ec..0000000000 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/reporting/PerformanceMetricsUtil.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.reporting; - -import org.apache.nifi.controller.ProcessorNode; -import org.apache.nifi.controller.repository.FlowFileEvent; -import org.apache.nifi.controller.status.ProcessingPerformanceStatus; - -public class PerformanceMetricsUtil { - - public static ProcessingPerformanceStatus getPerformanceMetrics(final FlowFileEvent fileEvent, final ProcessorNode processorNode) { - final ProcessingPerformanceStatus newMetrics = new ProcessingPerformanceStatus(); - - newMetrics.setIdentifier(processorNode.getProcessGroup().getIdentifier()); - newMetrics.setCpuDuration(fileEvent.getCpuNanoseconds()); - newMetrics.setContentReadDuration(fileEvent.getContentReadNanoseconds()); - newMetrics.setContentWriteDuration(fileEvent.getContentWriteNanoseconds()); - newMetrics.setSessionCommitDuration(fileEvent.getSessionCommitNanoseconds()); - newMetrics.setGarbageCollectionDuration(fileEvent.getGargeCollectionMillis()); - - return newMetrics; - } -} \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java index 2ba20f5759..91eee309dd 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java @@ -107,7 +107,6 @@ import org.apache.nifi.controller.state.SortedStateUtils; import org.apache.nifi.controller.status.ConnectionStatus; import org.apache.nifi.controller.status.PortStatus; import org.apache.nifi.controller.status.ProcessGroupStatus; -import org.apache.nifi.controller.status.ProcessingPerformanceStatus; import org.apache.nifi.controller.status.ProcessorStatus; import org.apache.nifi.controller.status.RemoteProcessGroupStatus; import org.apache.nifi.controller.status.analytics.ConnectionStatusPredictions; @@ -220,7 +219,6 @@ import org.apache.nifi.web.api.dto.status.PortStatusDTO; import org.apache.nifi.web.api.dto.status.PortStatusSnapshotDTO; import org.apache.nifi.web.api.dto.status.ProcessGroupStatusDTO; import org.apache.nifi.web.api.dto.status.ProcessGroupStatusSnapshotDTO; -import org.apache.nifi.web.api.dto.status.ProcessingPerformanceStatusDTO; import org.apache.nifi.web.api.dto.status.ProcessorStatusDTO; import org.apache.nifi.web.api.dto.status.ProcessorStatusSnapshotDTO; import org.apache.nifi.web.api.dto.status.RemoteProcessGroupStatusDTO; @@ -1078,11 +1076,6 @@ public final class DtoFactory { snapshot.setActiveThreadCount(processGroupStatus.getActiveThreadCount()); snapshot.setTerminatedThreadCount(processGroupStatus.getTerminatedThreadCount()); - final ProcessingPerformanceStatus performanceStatus = processGroupStatus.getProcessingPerformanceStatus(); - if (performanceStatus != null) { - snapshot.setProcessingPerformanceStatus(createProcessingPerformanceStatusDTO(performanceStatus)); - } - StatusMerger.updatePrettyPrintedFields(snapshot); return processGroupStatusDto; } @@ -1300,11 +1293,6 @@ public final class DtoFactory { snapshot.setTerminatedThreadCount(procStatus.getTerminatedThreadCount()); snapshot.setType(procStatus.getType()); - final ProcessingPerformanceStatus performanceStatus = procStatus.getProcessingPerformanceStatus(); - if (performanceStatus != null) { - snapshot.setProcessingPerformanceStatus(createProcessingPerformanceStatusDTO(procStatus.getProcessingPerformanceStatus())); - } - StatusMerger.updatePrettyPrintedFields(snapshot); return dto; } @@ -4972,18 +4960,4 @@ public final class DtoFactory { public void setExtensionManager(ExtensionManager extensionManager) { this.extensionManager = extensionManager; } - - private ProcessingPerformanceStatusDTO createProcessingPerformanceStatusDTO(final ProcessingPerformanceStatus performanceStatus) { - - final ProcessingPerformanceStatusDTO performanceStatusDTO = new ProcessingPerformanceStatusDTO(); - - performanceStatusDTO.setIdentifier(performanceStatus.getIdentifier()); - performanceStatusDTO.setCpuDuration(performanceStatus.getCpuDuration()); - performanceStatusDTO.setContentReadDuration(performanceStatusDTO.getContentReadDuration()); - performanceStatusDTO.setContentWriteDuration(performanceStatus.getContentWriteDuration()); - performanceStatusDTO.setSessionCommitDuration(performanceStatus.getSessionCommitDuration()); - performanceStatusDTO.setGarbageCollectionDuration(performanceStatus.getGarbageCollectionDuration()); - - return performanceStatusDTO; - } } diff --git a/nifi-nar-bundles/nifi-grpc-bundle/nifi-grpc-common/pom.xml b/nifi-nar-bundles/nifi-grpc-bundle/nifi-grpc-common/pom.xml new file mode 100644 index 0000000000..3150bc4cf8 --- /dev/null +++ b/nifi-nar-bundles/nifi-grpc-bundle/nifi-grpc-common/pom.xml @@ -0,0 +1,148 @@ + + + + 4.0.0 + + org.apache.nifi + nifi-grpc-bundle + 1.28.0-SNAPSHOT + + + nifi-grpc-common + + + + org.apache.nifi + nifi-api + + + org.apache.nifi + nifi-utils + 1.28.0-SNAPSHOT + + + org.apache.nifi + nifi-ssl-context-service-api + + + org.apache.nifi + nifi-security-utils-api + 1.28.0-SNAPSHOT + + + org.apache.commons + commons-lang3 + + + commons-io + commons-io + + + io.grpc + grpc-protobuf + ${grpc.version} + + + io.grpc + grpc-stub + ${grpc.version} + + + io.grpc + grpc-netty + ${grpc.version} + + + org.apache.nifi + nifi-security-utils + 1.28.0-SNAPSHOT + test + + + + + + + kr.motd.maven + os-maven-plugin + 1.6.2 + + + + + + org.apache.maven.plugins + maven-checkstyle-plugin + + **/grpc/FlowFileRequest.java,**/grpc/FlowFileReply.java,**/grpc/FFSProto.java,**/grpc/FlowFileServiceGrpc.java + + + + org.xolstice.maven.plugins + protobuf-maven-plugin + 0.6.1 + + com.google.protobuf:protoc:${protoc.version}:exe:${os.detected.classifier} + grpc-java + io.grpc:protoc-gen-grpc-java:${grpc.version}:exe:${os.detected.classifier} + ${basedir}/src/main/resources/proto + + + + + compile + compile-custom + + + + + + + org.codehaus.mojo + build-helper-maven-plugin + 3.4.0 + + + test + generate-sources + + add-source + + + + ${basedir}/target/generated-sources + + + + + + + + + + jigsaw + + (1.8,) + + + + javax.annotation + javax.annotation-api + + + + + diff --git a/nifi-nar-bundles/nifi-grpc-bundle/nifi-grpc-common/src/main/java/org/apache/nifi/processors/grpc/FlowFileIngestService.java b/nifi-nar-bundles/nifi-grpc-bundle/nifi-grpc-common/src/main/java/org/apache/nifi/processors/grpc/FlowFileIngestService.java new file mode 100644 index 0000000000..d14cf0c34d --- /dev/null +++ b/nifi-nar-bundles/nifi-grpc-bundle/nifi-grpc-common/src/main/java/org/apache/nifi/processors/grpc/FlowFileIngestService.java @@ -0,0 +1,163 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.grpc; + +import com.google.common.collect.Maps; +import com.google.protobuf.ByteString; +import io.grpc.stub.StreamObserver; +import org.apache.commons.io.IOUtils; +import org.apache.nifi.flowfile.FlowFile; +import org.apache.nifi.flowfile.attributes.CoreAttributes; +import org.apache.nifi.logging.ComponentLog; +import org.apache.nifi.processor.ProcessSession; +import org.apache.nifi.processor.ProcessSessionFactory; +import org.apache.nifi.processor.Relationship; +import org.apache.nifi.processors.grpc.util.BackpressureChecker; + +import java.io.BufferedOutputStream; +import java.io.InputStream; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; + +import static java.util.Objects.requireNonNull; + +/** + * Simple gRPC service that handles receipt of FlowFileRequest messages from external-to-NiFi gRPC + * clients. + */ +// NOTE: you may need to add the sources generated after running `maven clean compile` to your IDE +// configured source directories. Otherwise, the classes generated when the proto is compiled won't +// be accessible from here. For IntelliJ, open this module's settings and mark the following as source directories: +// +// * target/generated-sources/protobuf/grpc-java +// * target/generated-sources/protobuf/java +public class FlowFileIngestService extends FlowFileServiceGrpc.FlowFileServiceImplBase { + public static final String SERVICE_NAME = "grpc://FlowFileIngestService"; + public static final int FILES_BEFORE_CHECKING_DESTINATION_SPACE = 5; + + private final AtomicReference sessionFactoryReference; + private final ComponentLog logger; + private final Relationship relSuccess; + private final BackpressureChecker backpressureChecker; + + /** + * Create a FlowFileIngestService + * + * @param sessionFactoryReference a reference to a {@link ProcessSessionFactory} to route {@link + * FlowFile}s to process relationships. + */ + public FlowFileIngestService(final ComponentLog logger, + final AtomicReference sessionFactoryReference, + final Relationship relSuccess, + final BackpressureChecker backpressureChecker) { + this.sessionFactoryReference = requireNonNull(sessionFactoryReference); + this.logger = requireNonNull(logger); + this.relSuccess = requireNonNull(relSuccess); + this.backpressureChecker = requireNonNull(backpressureChecker); + } + + /** + * Handle receipt of a FlowFileRequest and route it to the appropriate process relationship. + * + * @param request the flowfile request + * @param responseObserver the mechanism by which to reply to the client + */ + @Override + public void send(final FlowFileRequest request, final StreamObserver responseObserver) { + final FlowFileReply.Builder replyBuilder = FlowFileReply.newBuilder(); + + final String remoteHost = FlowFileIngestServiceInterceptor.REMOTE_HOST_KEY.get(); + final String remoteDN = FlowFileIngestServiceInterceptor.REMOTE_DN_KEY.get(); + + // block until we have a session factory (occurs when processor is triggered) + ProcessSessionFactory sessionFactory = null; + while (sessionFactory == null) { + sessionFactory = sessionFactoryReference.get(); + if (sessionFactory == null) { + try { + Thread.sleep(10); + } catch (final InterruptedException e) { + } + } + } + + final ProcessSession session = sessionFactory.createSession(); + + // if there's no space available, reject the request. + if (backpressureChecker.isBackpressured()) { + final String message = "Received request from " + remoteHost + " but no space available; Indicating Service Unavailable"; + logger.debug(message); + final FlowFileReply reply = replyBuilder.setResponseCode(FlowFileReply.ResponseCode.ERROR) + .setBody(message) + .build(); + responseObserver.onNext(reply); + responseObserver.onCompleted(); + return; + } + + if (logger.isDebugEnabled()) { + logger.debug("Received request from " + remoteHost); + } + + final long startNanos = System.nanoTime(); + FlowFile flowFile = session.create(); + // push the attributes provided onto the created flowfile + final Map attributes = Maps.newHashMap(); + attributes.putAll(request.getAttributesMap()); + String sourceSystemFlowFileIdentifier = attributes.get(CoreAttributes.UUID.key()); + if (sourceSystemFlowFileIdentifier != null) { + sourceSystemFlowFileIdentifier = "urn:nifi:" + sourceSystemFlowFileIdentifier; + + // If we receveied a UUID, we want to give the FlowFile a new UUID and register the sending system's + // identifier as the SourceSystemFlowFileIdentifier field in the Provenance RECEIVE event + attributes.put(CoreAttributes.UUID.key(), UUID.randomUUID().toString()); + } + flowFile = session.putAllAttributes(flowFile, attributes); + final ByteString content = request.getContent(); + final InputStream contentStream = content.newInput(); + + // write the provided content to the flowfile + flowFile = session.write(flowFile, out -> { + try (final BufferedOutputStream bos = new BufferedOutputStream(out, 65536)) { + IOUtils.copy(contentStream, bos); + } + }); + final long transferNanos = System.nanoTime() - startNanos; + final long transferMillis = TimeUnit.MILLISECONDS.convert(transferNanos, TimeUnit.NANOSECONDS); + + session.getProvenanceReporter().receive(flowFile, + SERVICE_NAME, + sourceSystemFlowFileIdentifier, + "Remote DN=" + remoteDN, + transferMillis); + flowFile = session.putAttribute(flowFile, GRPCAttributeNames.REMOTE_HOST, remoteHost); + flowFile = session.putAttribute(flowFile, GRPCAttributeNames.REMOTE_USER_DN, remoteDN); + + // register success + session.transfer(flowFile, relSuccess); + session.commit(); + + // reply to client + final FlowFileReply reply = replyBuilder.setResponseCode(FlowFileReply.ResponseCode.SUCCESS) + .setBody("FlowFile successfully received.") + .build(); + responseObserver.onNext(reply); + responseObserver.onCompleted(); + } +} diff --git a/nifi-nar-bundles/nifi-grpc-bundle/nifi-grpc-common/src/main/java/org/apache/nifi/processors/grpc/FlowFileIngestServiceInterceptor.java b/nifi-nar-bundles/nifi-grpc-bundle/nifi-grpc-common/src/main/java/org/apache/nifi/processors/grpc/FlowFileIngestServiceInterceptor.java new file mode 100644 index 0000000000..6614e9be48 --- /dev/null +++ b/nifi-nar-bundles/nifi-grpc-bundle/nifi-grpc-common/src/main/java/org/apache/nifi/processors/grpc/FlowFileIngestServiceInterceptor.java @@ -0,0 +1,161 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.grpc; + +import io.grpc.Attributes; +import io.grpc.Context; +import io.grpc.Contexts; +import io.grpc.Grpc; +import io.grpc.Metadata; +import io.grpc.ServerCall; +import io.grpc.ServerCallHandler; +import io.grpc.ServerInterceptor; +import io.grpc.Status; +import org.apache.nifi.logging.ComponentLog; +import org.apache.nifi.processor.exception.ProcessException; + +import javax.net.ssl.SSLPeerUnverifiedException; +import javax.net.ssl.SSLSession; +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.security.cert.Certificate; +import java.security.cert.X509Certificate; +import java.util.regex.Pattern; + +import static java.util.Objects.requireNonNull; + +/** + * Simple gRPC service call interceptor that enforces various controls. + * Despite its name, it does not contain any FlowFileIngestService specific logic. + */ +public class FlowFileIngestServiceInterceptor implements ServerInterceptor { + + public static final String DEFAULT_FOUND_SUBJECT = "none"; + private static final String UNKNOWN_IP = "unknown-ip"; + private static final String DN_UNAUTHORIZED = "The client DN does not have permission to send gRPC requests to this NiFi. "; + private static final ServerCall.Listener IDENTITY_LISTENER = new ServerCall.Listener(){}; + + public static final Context.Key REMOTE_HOST_KEY = Context.key(GRPCAttributeNames.REMOTE_HOST); + public static final Context.Key REMOTE_DN_KEY = Context.key(GRPCAttributeNames.REMOTE_USER_DN); + + private final ComponentLog logger; + private Pattern authorizedDNPattern; + + /** + * Create an interceptor that applies various controls per request + * + * @param logger the {@link ComponentLog} for the ListenGRPC processor + */ + public FlowFileIngestServiceInterceptor(final ComponentLog logger) { + this.logger = requireNonNull(logger); + } + + /** + * Enforce that the requestor DN matches the provided pattern. + * + * @param authorizedDNPattern the pattern which DNs must match + * + * @return this + */ + public FlowFileIngestServiceInterceptor enforceDNPattern(final Pattern authorizedDNPattern) { + this.authorizedDNPattern = requireNonNull(authorizedDNPattern); + return this; + } + + /** + * Intercept incoming and outgoing messages and enforce any necessary controls + * + * @param call the request message + * @param headers the request metadata + * @param next the next interceptor in the interceptor chain prior to the service implementation + * @param The message request type (e.g. ReqT) + * @param The message reply type (e.g. RespT) + * + * @return a listener for the incoming call. + */ + @Override + public ServerCall.Listener interceptCall( + final ServerCall call, + final Metadata headers, + final ServerCallHandler next) { + + final Attributes attributes = call.getAttributes(); + final SocketAddress socketAddress = attributes.get(Grpc.TRANSPORT_ATTR_REMOTE_ADDR); + final String clientIp = clientIp(socketAddress); + String foundSubject = DEFAULT_FOUND_SUBJECT; + + // enforce that the DN on the client cert matches the configured pattern + final SSLSession sslSession = attributes.get(Grpc.TRANSPORT_ATTR_SSL_SESSION); + if (this.authorizedDNPattern != null && sslSession != null) { + try { + final Certificate[] certs = sslSession.getPeerCertificates(); + if (certs != null && certs.length > 0) { + for (final Certificate cert : certs) { + final X509Certificate x509Cert = toX509Certificate(cert); + foundSubject = x509Cert.getSubjectX500Principal().getName(); + if (authorizedDNPattern.matcher(foundSubject).matches()) { + break; + } else { + logger.warn("Rejecting transfer attempt from " + foundSubject + " because the DN is not authorized, host=" + clientIp); + call.close(Status.PERMISSION_DENIED.withDescription(DN_UNAUTHORIZED + foundSubject), headers); + return IDENTITY_LISTENER; + } + } + } + } catch (final SSLPeerUnverifiedException e) { + logger.debug("Skipping DN authorization for request from {}", clientIp, e); + } + } + // contextualize the DN and IP for use in the RPC implementation + final Context context = Context.current() + .withValue(REMOTE_HOST_KEY, clientIp) + .withValue(REMOTE_DN_KEY, foundSubject); + + // if we got to this point, there were no errors, call the next interceptor in the chain + return Contexts.interceptCall(context, call, headers, next); + } + + /** + * Grabs the client IP from the socket address pulled from the request metadata, or UNKNOWN + * if it's not possible to determine. + * + * @param socketAddress the socket address pulled from the gRPC request + * @return the client IP + */ + private String clientIp(final SocketAddress socketAddress) { + if (socketAddress == null) { + return UNKNOWN_IP; + } + + if (!(socketAddress instanceof InetSocketAddress)) { + return socketAddress.toString(); + } + + final InetSocketAddress inetSocketAddress = (InetSocketAddress) socketAddress; + final String hostString = inetSocketAddress.getHostString(); + return hostString == null ? UNKNOWN_IP : hostString; + } + + private X509Certificate toX509Certificate(final Certificate certificate) { + if (certificate instanceof X509Certificate) { + return (X509Certificate) certificate; + } else { + throw new ProcessException("Certificate is not an X.509 certificate. Certificate type: " + certificate.getClass()); + } + } + +} diff --git a/nifi-nar-bundles/nifi-grpc-bundle/nifi-grpc-common/src/main/java/org/apache/nifi/processors/grpc/GRPCAttributeNames.java b/nifi-nar-bundles/nifi-grpc-bundle/nifi-grpc-common/src/main/java/org/apache/nifi/processors/grpc/GRPCAttributeNames.java new file mode 100644 index 0000000000..a2f25cf55f --- /dev/null +++ b/nifi-nar-bundles/nifi-grpc-bundle/nifi-grpc-common/src/main/java/org/apache/nifi/processors/grpc/GRPCAttributeNames.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.grpc; + +public class GRPCAttributeNames { + + public static final String REMOTE_USER_DN = "listengrpc.remote.user.dn"; + public static final String REMOTE_HOST = "listengrpc.remote.host"; +} diff --git a/nifi-nar-bundles/nifi-grpc-bundle/nifi-grpc-common/src/main/java/org/apache/nifi/processors/grpc/ssl/SslContextProvider.java b/nifi-nar-bundles/nifi-grpc-bundle/nifi-grpc-common/src/main/java/org/apache/nifi/processors/grpc/ssl/SslContextProvider.java new file mode 100644 index 0000000000..1833970b5c --- /dev/null +++ b/nifi-nar-bundles/nifi-grpc-bundle/nifi-grpc-common/src/main/java/org/apache/nifi/processors/grpc/ssl/SslContextProvider.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.grpc.ssl; + +import io.netty.handler.codec.http2.Http2SecurityUtil; +import io.netty.handler.ssl.ApplicationProtocolConfig; +import io.netty.handler.ssl.ClientAuth; +import io.netty.handler.ssl.JdkSslContext; +import io.netty.handler.ssl.SslContext; +import io.netty.handler.ssl.SupportedCipherSuiteFilter; +import org.apache.commons.lang3.StringUtils; +import org.apache.nifi.security.util.TlsConfiguration; +import org.apache.nifi.ssl.SSLContextService; + +import javax.net.ssl.SSLContext; + +/** + * Provider for Netty SslContext from NiFi SSLContextService + */ +public class SslContextProvider { + private static final boolean START_TLS = false; + + private static final String H2_PROTOCOL = "h2"; + + public static SslContext getSslContext(final SSLContextService sslContextService, final boolean client) { + final SSLContext sslContext = sslContextService.createContext(); + final TlsConfiguration tlsConfiguration = sslContextService.createTlsConfiguration(); + final ClientAuth clientAuth = StringUtils.isBlank(tlsConfiguration.getTruststorePath()) ? ClientAuth.NONE : ClientAuth.REQUIRE; + + final ApplicationProtocolConfig applicationProtocolConfig = new ApplicationProtocolConfig( + ApplicationProtocolConfig.Protocol.ALPN, + ApplicationProtocolConfig.SelectorFailureBehavior.NO_ADVERTISE, + ApplicationProtocolConfig.SelectedListenerFailureBehavior.ACCEPT, + H2_PROTOCOL + ); + + return new JdkSslContext( + sslContext, + client, + Http2SecurityUtil.CIPHERS, + SupportedCipherSuiteFilter.INSTANCE, + applicationProtocolConfig, + clientAuth, + tlsConfiguration.getEnabledProtocols(), + START_TLS + ); + } +} diff --git a/nifi-nar-bundles/nifi-grpc-bundle/nifi-grpc-common/src/main/java/org/apache/nifi/processors/grpc/util/BackpressureChecker.java b/nifi-nar-bundles/nifi-grpc-bundle/nifi-grpc-common/src/main/java/org/apache/nifi/processors/grpc/util/BackpressureChecker.java new file mode 100644 index 0000000000..064b91e588 --- /dev/null +++ b/nifi-nar-bundles/nifi-grpc-bundle/nifi-grpc-common/src/main/java/org/apache/nifi/processors/grpc/util/BackpressureChecker.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.grpc.util; + +import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.processor.Relationship; + +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; + +public class BackpressureChecker { + + private static final int RECHECK_THRESHOLD = 5; + + private final ProcessContext processContext; + private final Set relationships; + private final AtomicLong requestCount = new AtomicLong(0L); + private final AtomicBoolean backPressure = new AtomicBoolean(false); + + public BackpressureChecker(ProcessContext processContext, Set relationships) { + this.processContext = processContext; + this.relationships = relationships; + } + + public boolean isBackpressured() { + long n = requestCount.getAndIncrement() % RECHECK_THRESHOLD; + if (n == 0 || backPressure.get()) { + backPressure.set(processContext.getAvailableRelationships().size() != relationships.size()); + } + return backPressure.get(); + } +} diff --git a/nifi-nar-bundles/nifi-grpc-bundle/nifi-grpc-common/src/main/resources/proto/flowfile_service.proto b/nifi-nar-bundles/nifi-grpc-bundle/nifi-grpc-common/src/main/resources/proto/flowfile_service.proto new file mode 100644 index 0000000000..1eb18a5d7e --- /dev/null +++ b/nifi-nar-bundles/nifi-grpc-bundle/nifi-grpc-common/src/main/resources/proto/flowfile_service.proto @@ -0,0 +1,59 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// NOTE: you may need to add the sources generated when running `maven clean compile` to your IDE +// configured source directories. Otherwise, the classes generated when the proto is compiled won't +// be accessible. For IntelliJ, open this module's settings and mark the following as source directories: +// +// * target/generated-sources/protobuf/grpc-java +// * target/generated-sources/protobuf/java + +syntax = "proto3"; + +option java_multiple_files = true; +option java_package = "org.apache.nifi.processors.grpc"; +option java_outer_classname = "FFSProto"; +option objc_class_prefix = "FFS"; + +package org.apache.nifi.processors.grpc; + +// The FlowFile service definition. +service FlowFileService { + // Sends a FlowFile (blocking rpc) + rpc Send (FlowFileRequest) returns (FlowFileReply) {} +} + +// The request message +message FlowFileRequest { + // tags 1-15 require one byte to encode and should be left for commonly occurring tags. + // For that reason, tags 3-14 are left available. + // + // source: https://developers.google.com/protocol-buffers/docs/proto3#assigning-tags + int64 id = 1; + map attributes = 2; + bytes content = 15; +} + +// the reply message +message FlowFileReply { + enum ResponseCode { + ERROR = 0; + SUCCESS = 1; + RETRY = 2; + } + + ResponseCode responseCode = 1; + string body = 2; +} \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-grpc-bundle/nifi-grpc-common/src/test/java/org/apache/nifi/processors/grpc/ssl/SslContextProviderTest.java b/nifi-nar-bundles/nifi-grpc-bundle/nifi-grpc-common/src/test/java/org/apache/nifi/processors/grpc/ssl/SslContextProviderTest.java new file mode 100644 index 0000000000..4d5f06f63d --- /dev/null +++ b/nifi-nar-bundles/nifi-grpc-bundle/nifi-grpc-common/src/test/java/org/apache/nifi/processors/grpc/ssl/SslContextProviderTest.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.grpc.ssl; + +import io.netty.buffer.ByteBufAllocator; +import io.netty.handler.ssl.SslContext; +import org.apache.nifi.security.util.SslContextFactory; +import org.apache.nifi.security.util.StandardTlsConfiguration; +import org.apache.nifi.security.util.TemporaryKeyStoreBuilder; +import org.apache.nifi.security.util.TlsConfiguration; +import org.apache.nifi.security.util.TlsException; +import org.apache.nifi.ssl.RestrictedSSLContextService; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLEngine; + +import java.util.Collections; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class SslContextProviderTest { + static final List APPLICATION_PROTOCOLS = Collections.singletonList("h2"); + + static TlsConfiguration tlsConfiguration; + + static SSLContext sslContext; + + @Mock + RestrictedSSLContextService sslContextService; + + @BeforeAll + static void setTlsConfiguration() throws TlsException { + tlsConfiguration = new TemporaryKeyStoreBuilder().build(); + sslContext = SslContextFactory.createSslContext(tlsConfiguration); + } + + @Test + void testGetClientSslContext() { + when(sslContextService.createContext()).thenReturn(sslContext); + when(sslContextService.createTlsConfiguration()).thenReturn(tlsConfiguration); + + final SslContext clientSslContext = SslContextProvider.getSslContext(sslContextService, true); + + assertNotNull(clientSslContext); + assertTrue(clientSslContext.isClient()); + assertFalse(clientSslContext.isServer()); + assertEquals(APPLICATION_PROTOCOLS, clientSslContext.applicationProtocolNegotiator().protocols()); + } + + @Test + void testGetServerSslContextClientAuthRequired() { + when(sslContextService.createContext()).thenReturn(sslContext); + when(sslContextService.createTlsConfiguration()).thenReturn(tlsConfiguration); + + final SslContext serverSslContext = SslContextProvider.getSslContext(sslContextService, false); + + assertServerStatus(serverSslContext); + + final SSLEngine sslEngine = serverSslContext.newEngine(ByteBufAllocator.DEFAULT); + assertTrue(sslEngine.getNeedClientAuth()); + } + + @Test + void testGetServerSslContextClientAuthNone() { + when(sslContextService.createContext()).thenReturn(sslContext); + + final TlsConfiguration keyStoreConfiguration = new StandardTlsConfiguration(); + when(sslContextService.createTlsConfiguration()).thenReturn(keyStoreConfiguration); + + final SslContext serverSslContext = SslContextProvider.getSslContext(sslContextService, false); + + assertServerStatus(serverSslContext); + + final SSLEngine sslEngine = serverSslContext.newEngine(ByteBufAllocator.DEFAULT); + assertFalse(sslEngine.getNeedClientAuth()); + } + + private void assertServerStatus(final SslContext configuredSslContext) { + assertNotNull(configuredSslContext); + assertFalse(configuredSslContext.isClient()); + assertTrue(configuredSslContext.isServer()); + assertEquals(APPLICATION_PROTOCOLS, configuredSslContext.applicationProtocolNegotiator().protocols()); + } +} diff --git a/nifi-nar-bundles/nifi-grpc-bundle/nifi-grpc-nar/pom.xml b/nifi-nar-bundles/nifi-grpc-bundle/nifi-grpc-nar/pom.xml new file mode 100644 index 0000000000..1ebb04019e --- /dev/null +++ b/nifi-nar-bundles/nifi-grpc-bundle/nifi-grpc-nar/pom.xml @@ -0,0 +1,41 @@ + + + + 4.0.0 + + + org.apache.nifi + nifi-grpc-bundle + 1.28.0-SNAPSHOT + + + nifi-grpc-nar + nar + + + + org.apache.nifi + nifi-standard-services-api-nar + 1.28.0-SNAPSHOT + nar + + + org.apache.nifi + nifi-grpc-processors + 1.28.0-SNAPSHOT + + + diff --git a/nifi-nar-bundles/nifi-grpc-bundle/nifi-grpc-nar/src/main/resources/LICENSE b/nifi-nar-bundles/nifi-grpc-bundle/nifi-grpc-nar/src/main/resources/LICENSE new file mode 100644 index 0000000000..7bb71ec9b6 --- /dev/null +++ b/nifi-nar-bundles/nifi-grpc-bundle/nifi-grpc-nar/src/main/resources/LICENSE @@ -0,0 +1,259 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +The binary distribution of this product bundles 'Google Protocol Buffers Java 3.3.1' +which is licensed under a BSD license. + + This license applies to all parts of Protocol Buffers except the following: + + - Atomicops support for generic gcc, located in + src/google/protobuf/stubs/atomicops_internals_generic_gcc.h. + This file is copyrighted by Red Hat Inc. + + - Atomicops support for AIX/POWER, located in + src/google/protobuf/stubs/atomicops_internals_power.h. + This file is copyrighted by Bloomberg Finance LP. + + Copyright 2014, Google Inc. All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + * Neither the name of Google Inc. nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + Code generated by the Protocol Buffer compiler is owned by the owner + of the input file used when generating it. This code is not + standalone and requires a support library to be linked with it. This + support library is itself covered by the above license. + +The binary distribution of this product bundles 'The JSR-305 reference implementation' +which is licensed under a BSD license. + + The JSR-305 reference implementation (lib/jsr305.jar) is + distributed under the terms of the New BSD license: + + http://www.opensource.org/licenses/bsd-license.php + + See the JSR-305 home page for more information: + + http://code.google.com/p/jsr-305/ \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-grpc-bundle/nifi-grpc-nar/src/main/resources/NOTICE b/nifi-nar-bundles/nifi-grpc-bundle/nifi-grpc-nar/src/main/resources/NOTICE new file mode 100644 index 0000000000..ce543d6c62 --- /dev/null +++ b/nifi-nar-bundles/nifi-grpc-bundle/nifi-grpc-nar/src/main/resources/NOTICE @@ -0,0 +1,295 @@ +nifi-grpc-nar +Copyright 2017-2020 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +****************** +Apache Software License v2 +****************** + +The following binary components are provided under the Apache Software License v2 + + (ASLv2) Apache Commons Lang + The following NOTICE information applies: + Apache Commons Lang + Copyright 2001-2014 The Apache Software Foundation + + This product includes software from the Spring Framework, + under the Apache License 2.0 (see: StringUtils.containsWhitespace()) + + (ASLv2) gRPC-Java + The following NOTICE information applies: + Copyright 2014, gRPC Authors All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + ----------------------------------------------------------------------- + + This product contains a modified portion of 'OkHttp', an open source + HTTP & SPDY client for Android and Java applications, which can be obtained + at: + + * LICENSE: + * okhttp/third_party/okhttp/LICENSE (Apache License 2.0) + * HOMEPAGE: + * https://github.com/square/okhttp + * LOCATION_IN_GRPC: + * okhttp/third_party/okhttp + + This product contains a modified portion of 'Netty', an open source + networking library, which can be obtained at: + + * LICENSE: + * netty/third_party/netty/LICENSE.txt (Apache License 2.0) + * HOMEPAGE: + * https://netty.io + * LOCATION_IN_GRPC: + * netty/third_party/netty + + (ASLv2) The Netty Project + The following NOTICE information applies: + The Netty Project + ================= + + Please visit the Netty web site for more information: + + * http://netty.io/ + + Copyright 2014 The Netty Project + + The Netty Project licenses this file to you under the Apache License, + version 2.0 (the "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at: + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + + Also, please refer to each LICENSE..txt file, which is located in + the 'license' directory of the distribution file, for the license terms of the + components that this product depends on. + + ------------------------------------------------------------------------------- + This product contains the extensions to Java Collections Framework which has + been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: + + * LICENSE: + * license/LICENSE.jsr166y.txt (Public Domain) + * HOMEPAGE: + * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/ + * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/ + + This product contains a modified version of Robert Harder's Public Domain + Base64 Encoder and Decoder, which can be obtained at: + + * LICENSE: + * license/LICENSE.base64.txt (Public Domain) + * HOMEPAGE: + * http://iharder.sourceforge.net/current/java/base64/ + + This product contains a modified portion of 'Webbit', an event based + WebSocket and HTTP server, which can be obtained at: + + * LICENSE: + * license/LICENSE.webbit.txt (BSD License) + * HOMEPAGE: + * https://github.com/joewalnes/webbit + + This product contains a modified portion of 'SLF4J', a simple logging + facade for Java, which can be obtained at: + + * LICENSE: + * license/LICENSE.slf4j.txt (MIT License) + * HOMEPAGE: + * http://www.slf4j.org/ + + This product contains a modified portion of 'Apache Harmony', an open source + Java SE, which can be obtained at: + + * LICENSE: + * license/LICENSE.harmony.txt (Apache License 2.0) + * HOMEPAGE: + * http://archive.apache.org/dist/harmony/ + + This product contains a modified portion of 'jbzip2', a Java bzip2 compression + and decompression library written by Matthew J. Francis. It can be obtained at: + + * LICENSE: + * license/LICENSE.jbzip2.txt (MIT License) + * HOMEPAGE: + * https://code.google.com/p/jbzip2/ + + This product contains a modified portion of 'libdivsufsort', a C API library to construct + the suffix array and the Burrows-Wheeler transformed string for any input string of + a constant-size alphabet written by Yuta Mori. It can be obtained at: + + * LICENSE: + * license/LICENSE.libdivsufsort.txt (MIT License) + * HOMEPAGE: + * https://github.com/y-256/libdivsufsort + + This product contains a modified portion of Nitsan Wakart's 'JCTools', Java Concurrency Tools for the JVM, + which can be obtained at: + + * LICENSE: + * license/LICENSE.jctools.txt (ASL2 License) + * HOMEPAGE: + * https://github.com/JCTools/JCTools + + This product optionally depends on 'JZlib', a re-implementation of zlib in + pure Java, which can be obtained at: + + * LICENSE: + * license/LICENSE.jzlib.txt (BSD style License) + * HOMEPAGE: + * http://www.jcraft.com/jzlib/ + + This product optionally depends on 'Compress-LZF', a Java library for encoding and + decoding data in LZF format, written by Tatu Saloranta. It can be obtained at: + + * LICENSE: + * license/LICENSE.compress-lzf.txt (Apache License 2.0) + * HOMEPAGE: + * https://github.com/ning/compress + + This product optionally depends on 'lz4', a LZ4 Java compression + and decompression library written by Adrien Grand. It can be obtained at: + + * LICENSE: + * license/LICENSE.lz4.txt (Apache License 2.0) + * HOMEPAGE: + * https://github.com/jpountz/lz4-java + + This product optionally depends on 'lzma-java', a LZMA Java compression + and decompression library, which can be obtained at: + + * LICENSE: + * license/LICENSE.lzma-java.txt (Apache License 2.0) + * HOMEPAGE: + * https://github.com/jponge/lzma-java + + This product contains a modified portion of 'jfastlz', a Java port of FastLZ compression + and decompression library written by William Kinney. It can be obtained at: + + * LICENSE: + * license/LICENSE.jfastlz.txt (MIT License) + * HOMEPAGE: + * https://code.google.com/p/jfastlz/ + + This product contains a modified portion of and optionally depends on 'Protocol Buffers', Google's data + interchange format, which can be obtained at: + + * LICENSE: + * license/LICENSE.protobuf.txt (New BSD License) + * HOMEPAGE: + * https://github.com/google/protobuf + + This product optionally depends on 'Bouncy Castle Crypto APIs' to generate + a temporary self-signed X.509 certificate when the JVM does not provide the + equivalent functionality. It can be obtained at: + + * LICENSE: + * license/LICENSE.bouncycastle.txt (MIT License) + * HOMEPAGE: + * http://www.bouncycastle.org/ + + This product optionally depends on 'Snappy', a compression library produced + by Google Inc, which can be obtained at: + + * LICENSE: + * license/LICENSE.snappy.txt (New BSD License) + * HOMEPAGE: + * https://github.com/google/snappy + + This product optionally depends on 'JBoss Marshalling', an alternative Java + serialization API, which can be obtained at: + + * LICENSE: + * license/LICENSE.jboss-marshalling.txt (GNU LGPL 2.1) + * HOMEPAGE: + * http://www.jboss.org/jbossmarshalling + + This product optionally depends on 'Caliper', Google's micro- + benchmarking framework, which can be obtained at: + + * LICENSE: + * license/LICENSE.caliper.txt (Apache License 2.0) + * HOMEPAGE: + * https://github.com/google/caliper + + This product optionally depends on 'Apache Log4J', a logging framework, which + can be obtained at: + + * LICENSE: + * license/LICENSE.log4j.txt (Apache License 2.0) + * HOMEPAGE: + * http://logging.apache.org/log4j/ + + This product optionally depends on 'Aalto XML', an ultra-high performance + non-blocking XML processor, which can be obtained at: + + * LICENSE: + * license/LICENSE.aalto-xml.txt (Apache License 2.0) + * HOMEPAGE: + * http://wiki.fasterxml.com/AaltoHome + + This product contains a modified version of 'HPACK', a Java implementation of + the HTTP/2 HPACK algorithm written by Twitter. It can be obtained at: + + * LICENSE: + * license/LICENSE.hpack.txt (Apache License 2.0) + * HOMEPAGE: + * https://github.com/twitter/hpack + + This product contains a modified portion of 'Apache Commons Lang', a Java library + provides utilities for the java.lang API, which can be obtained at: + + * LICENSE: + * license/LICENSE.commons-lang.txt (Apache License 2.0) + * HOMEPAGE: + * https://commons.apache.org/proper/commons-lang/ + + This product contains a forked and modified version of Tomcat Native + + * LICENSE: + * ASL2 + * HOMEPAGE: + * http://tomcat.apache.org/native-doc/ + * https://svn.apache.org/repos/asf/tomcat/native/ + + (ASLv2) Guava + The following NOTICE information applies: + Guava + Copyright 2015 The Guava Authors + + (ASLv2) Google GSON + The following NOTICE information applies: + Copyright 2008 Google Inc. + + (ASLv2) Error Prone + The following NOTICE information applies: + Copyright 2017 Google Inc. + + (ASLv2) Instrumentation + The following NOTICE information applies: + Copyright 2016 Google Inc. + + (ASLv2) Google APIs + The following NOTICE information applies: + Copyright 2016 Google Inc. + diff --git a/nifi-nar-bundles/nifi-grpc-bundle/nifi-grpc-processors/pom.xml b/nifi-nar-bundles/nifi-grpc-bundle/nifi-grpc-processors/pom.xml new file mode 100644 index 0000000000..51b707d034 --- /dev/null +++ b/nifi-nar-bundles/nifi-grpc-bundle/nifi-grpc-processors/pom.xml @@ -0,0 +1,46 @@ + + + + 4.0.0 + + + org.apache.nifi + nifi-grpc-bundle + 1.28.0-SNAPSHOT + + + nifi-grpc-processors + jar + + + + org.apache.nifi + nifi-grpc-common + 1.28.0-SNAPSHOT + + + org.apache.nifi + nifi-ssl-context-service-api + + + org.apache.nifi + nifi-mock + 1.28.0-SNAPSHOT + test + + + org.apache.nifi + nifi-ssl-context-service + test + + + diff --git a/nifi-nar-bundles/nifi-grpc-bundle/nifi-grpc-processors/src/main/java/org/apache/nifi/processors/grpc/InvokeGRPC.java b/nifi-nar-bundles/nifi-grpc-bundle/nifi-grpc-processors/src/main/java/org/apache/nifi/processors/grpc/InvokeGRPC.java new file mode 100644 index 0000000000..6811afd181 --- /dev/null +++ b/nifi-nar-bundles/nifi-grpc-bundle/nifi-grpc-processors/src/main/java/org/apache/nifi/processors/grpc/InvokeGRPC.java @@ -0,0 +1,438 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.grpc; + +import com.google.protobuf.ByteString; +import io.grpc.CompressorRegistry; +import io.grpc.DecompressorRegistry; +import io.grpc.ManagedChannel; +import io.grpc.netty.NettyChannelBuilder; +import io.netty.handler.ssl.SslContext; +import org.apache.nifi.annotation.behavior.EventDriven; +import org.apache.nifi.annotation.behavior.InputRequirement; +import org.apache.nifi.annotation.behavior.SupportsBatching; +import org.apache.nifi.annotation.behavior.WritesAttribute; +import org.apache.nifi.annotation.behavior.WritesAttributes; +import org.apache.nifi.annotation.documentation.CapabilityDescription; +import org.apache.nifi.annotation.documentation.Tags; +import org.apache.nifi.annotation.lifecycle.OnScheduled; +import org.apache.nifi.annotation.lifecycle.OnShutdown; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.components.ValidationContext; +import org.apache.nifi.components.ValidationResult; +import org.apache.nifi.flowfile.FlowFile; +import org.apache.nifi.logging.ComponentLog; +import org.apache.nifi.processor.AbstractProcessor; +import org.apache.nifi.processor.DataUnit; +import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.processor.ProcessSession; +import org.apache.nifi.processor.Relationship; +import org.apache.nifi.processor.exception.ProcessException; +import org.apache.nifi.processor.util.StandardValidators; +import org.apache.nifi.processors.grpc.ssl.SslContextProvider; +import org.apache.nifi.ssl.SSLContextService; +import org.apache.nifi.annotation.documentation.DeprecationNotice; + +import java.io.InputStream; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; + +@EventDriven +@SupportsBatching +@Tags({"grpc", "rpc", "client"}) +@InputRequirement(InputRequirement.Requirement.INPUT_REQUIRED) +@CapabilityDescription("Sends FlowFiles, optionally with content, to a configurable remote gRPC service endpoint. The remote gRPC service must abide by the service IDL defined in NiFi. " + + " gRPC isn't intended to carry large payloads, so this processor should be used only when FlowFile" + + " sizes are on the order of megabytes. The default maximum message size is 4MB.") +@WritesAttributes({ + @WritesAttribute(attribute = "invokegrpc.response.code", description = "The response code that is returned (0 = ERROR, 1 = SUCCESS, 2 = RETRY)"), + @WritesAttribute(attribute = "invokegrpc.response.body", description = "The response message that is returned"), + @WritesAttribute(attribute = "invokegrpc.service.host", description = "The remote gRPC service hostname"), + @WritesAttribute(attribute = "invokegrpc.service.port", description = "The remote gRPC service port"), + @WritesAttribute(attribute = "invokegrpc.java.exception.class", description = "The Java exception class raised when the processor fails"), + @WritesAttribute(attribute = "invokegrpc.java.exception.message", description = "The Java exception message raised when the processor fails"), +}) +@DeprecationNotice(reason = "No planned alternatives to be offered. Use custom processors instead.") +public class InvokeGRPC extends AbstractProcessor { + public static final String RESPONSE_CODE = "invokegrpc.response.code"; + public static final String RESPONSE_BODY = "invokegrpc.response.body"; + public static final String SERVICE_HOST = "invokegrpc.service.host"; + public static final String SERVICE_PORT = "invokegrpc.service.port"; + public static final String EXCEPTION_CLASS = "invokegrpc.java.exception.class"; + public static final String EXCEPTION_MESSAGE = "invokegrpc.java.exception.message"; + + // properties + public static final PropertyDescriptor PROP_SERVICE_HOST = new PropertyDescriptor.Builder() + .name("Remote gRPC service hostname") + .description("Remote host which will be connected to") + .required(true) + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .build(); + public static final PropertyDescriptor PROP_SERVICE_PORT = new PropertyDescriptor.Builder() + .name("Remote gRPC service port") + .description("Remote port which will be connected to") + .required(true) + .addValidator(StandardValidators.PORT_VALIDATOR) + .build(); + public static final PropertyDescriptor PROP_MAX_MESSAGE_SIZE = new PropertyDescriptor.Builder() + .name("Max Message Size") + .description("The maximum size of FlowFiles that this processor will allow to be received." + + " The default is 4MB. If FlowFiles exceed this size, you should consider using another transport mechanism" + + " as gRPC isn't designed for heavy payloads.") + .defaultValue("4MB") + .required(false) + .addValidator(StandardValidators.DATA_SIZE_VALIDATOR) + .build(); + public static final PropertyDescriptor PROP_USE_SECURE = new PropertyDescriptor.Builder() + .name("Use SSL/TLS") + .displayName("Use TLS") + .description("Whether or not to use TLS to send the contents of the gRPC messages.") + .required(false) + .defaultValue("false") + .allowableValues("true", "false") + .build(); + public static final PropertyDescriptor PROP_SSL_CONTEXT_SERVICE = new PropertyDescriptor.Builder() + .name("SSL Context Service") + .description("The SSL Context Service used to provide client certificate information for TLS (https) connections.") + .required(false) + .identifiesControllerService(SSLContextService.class) + .dependsOn(PROP_USE_SECURE, "true") + .build(); + public static final PropertyDescriptor PROP_SEND_CONTENT = new PropertyDescriptor.Builder() + .name("Send FlowFile Content") + .description("Whether or not to include the FlowFile content in the FlowFileRequest to the gRPC service.") + .required(false) + .defaultValue("true") + .allowableValues("true", "false") + .build(); + public static final PropertyDescriptor PROP_PENALIZE_NO_RETRY = new PropertyDescriptor.Builder() + .name("Penalize on \"No Retry\"") + .description("Enabling this property will penalize FlowFiles that are routed to the \"No Retry\" relationship.") + .required(false) + .defaultValue("false") + .allowableValues("true", "false") + .build(); + public static final PropertyDescriptor PROP_OUTPUT_RESPONSE_REGARDLESS = new PropertyDescriptor.Builder() + .name("Always Output Response") + .description("Will force a response FlowFile to be generated and routed to the 'Response' relationship regardless of what the server status code received is " + + "or if the processor is configured to put the server response body in the request attribute. In the later configuration a request FlowFile with the " + + "response body in the attribute and a typical response FlowFile will be emitted to their respective relationships.") + .required(false) + .defaultValue("false") + .allowableValues("true", "false") + .build(); + public static final List PROPERTIES = Collections.unmodifiableList(Arrays.asList( + PROP_SERVICE_HOST, + PROP_SERVICE_PORT, + PROP_MAX_MESSAGE_SIZE, + PROP_USE_SECURE, + PROP_SSL_CONTEXT_SERVICE, + PROP_SEND_CONTENT, + PROP_OUTPUT_RESPONSE_REGARDLESS, + PROP_PENALIZE_NO_RETRY + )); + + // relationships + public static final Relationship REL_SUCCESS_REQ = new Relationship.Builder() + .name("Original") + .description("The original FlowFile will be routed upon success. It will have new attributes detailing the " + + "success of the request.") + .build(); + public static final Relationship REL_RESPONSE = new Relationship.Builder() + .name("Response") + .description("A Response FlowFile will be routed upon success. If the 'Output Response Regardless' property " + + "is true then the response will be sent to this relationship regardless of the status code received.") + .build(); + public static final Relationship REL_RETRY = new Relationship.Builder() + .name("Retry") + .description("The original FlowFile will be routed on any status code that can be retried. It will have new " + + "attributes detailing the request.") + .build(); + public static final Relationship REL_NO_RETRY = new Relationship.Builder() + .name("No Retry") + .description("The original FlowFile will be routed on any status code that should NOT be retried. " + + "It will have new attributes detailing the request.") + .build(); + public static final Relationship REL_FAILURE = new Relationship.Builder() + .name("Failure") + .description("The original FlowFile will be routed on any type of connection failure, timeout or general exception. " + + "It will have new attributes detailing the request.") + .build(); + public static final Set RELATIONSHIPS = Collections.unmodifiableSet(new HashSet<>(Arrays.asList( + REL_SUCCESS_REQ, + REL_NO_RETRY, + REL_RESPONSE, + REL_RETRY, + REL_FAILURE + ))); + + private static final String USER_AGENT_PREFIX = "NiFi_invokeGRPC"; + // NOTE: you may need to add the sources generated after running `maven clean compile` to your IDE + // configured source directories. Otherwise, the classes generated when the proto is compiled won't + // be accessible from here. For IntelliJ, open this module's settings and mark the following as source directories: + // + // * target/generated-sources/protobuf/grpc-java + // * target/generated-sources/protobuf/java + private final AtomicReference blockingStubReference = new AtomicReference<>(); + private final AtomicReference channelReference = new AtomicReference<>(); + + @Override + protected List getSupportedPropertyDescriptors() { + return PROPERTIES; + } + + @Override + public Set getRelationships() { + return RELATIONSHIPS; + } + + @Override + protected Collection customValidate(ValidationContext context) { + List results = new ArrayList<>(super.customValidate(context)); + + final boolean useSecure = context.getProperty(PROP_USE_SECURE).asBoolean(); + final boolean sslContextServiceConfigured = context.getProperty(PROP_SSL_CONTEXT_SERVICE).isSet(); + + if (useSecure && !sslContextServiceConfigured) { + results.add(new ValidationResult.Builder() + .subject(PROP_SSL_CONTEXT_SERVICE.getDisplayName()) + .valid(false) + .explanation(String.format("'%s' must be configured when '%s' is true", PROP_SSL_CONTEXT_SERVICE.getDisplayName(), PROP_USE_SECURE.getDisplayName())) + .build()); + } + + return results; + } + + /** + * Whenever this processor is triggered, we need to construct a client in order to communicate + * with the configured gRPC service. + * + * @param context the processor context + */ + @OnScheduled + public void initializeClient(final ProcessContext context) { + + channelReference.set(null); + blockingStubReference.set(null); + final ComponentLog logger = getLogger(); + + final String host = context.getProperty(PROP_SERVICE_HOST).getValue(); + final int port = context.getProperty(PROP_SERVICE_PORT).asInteger(); + final int maxMessageSize = context.getProperty(PROP_MAX_MESSAGE_SIZE).asDataSize(DataUnit.B).intValue(); + String userAgent = USER_AGENT_PREFIX; + try { + userAgent += "_" + InetAddress.getLocalHost().getHostName(); + } catch (final UnknownHostException e) { + logger.warn("Unable to determine local hostname. Defaulting gRPC user agent to {}.", USER_AGENT_PREFIX, e); + } + + final NettyChannelBuilder nettyChannelBuilder = NettyChannelBuilder.forAddress(host, port) + // supports both gzip and plaintext, but will compress by default. + .compressorRegistry(CompressorRegistry.getDefaultInstance()) + .decompressorRegistry(DecompressorRegistry.getDefaultInstance()) + .maxInboundMessageSize(maxMessageSize) + .userAgent(userAgent); + + // configure whether or not we're using secure comms + final boolean useSecure = context.getProperty(PROP_USE_SECURE).asBoolean(); + final SSLContextService sslContextService = context.getProperty(PROP_SSL_CONTEXT_SERVICE).asControllerService(SSLContextService.class); + + if (useSecure) { + final SslContext clientSslContext = SslContextProvider.getSslContext(sslContextService, true); + nettyChannelBuilder.sslContext(clientSslContext); + } else { + nettyChannelBuilder.usePlaintext(); + } + + final ManagedChannel channel = nettyChannelBuilder.build(); + final FlowFileServiceGrpc.FlowFileServiceBlockingStub blockingStub = FlowFileServiceGrpc.newBlockingStub(channel); + channelReference.set(channel); + blockingStubReference.set(blockingStub); + } + + /** + * Perform cleanup prior to JVM shutdown + * + * @param context the processor context + * @throws InterruptedException if there's an issue cleaning up + */ + @OnShutdown + public void shutdown(final ProcessContext context) throws InterruptedException { + // close the channel + final ManagedChannel channel = channelReference.get(); + if (channel != null) { + channel.shutdown().awaitTermination(5, TimeUnit.SECONDS); + } + } + + @Override + public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException { + FlowFile fileToProcess = null; + if (context.hasIncomingConnection()) { + fileToProcess = session.get(); + + // If we have no FlowFile, and all incoming connections are self-loops then we can continue on. + // However, if we have no FlowFile and we have connections coming from other Processors, then + // we know that we should run only if we have a FlowFile. + if (fileToProcess == null && context.hasNonLoopConnection()) { + return; + } + } + + final ComponentLog logger = getLogger(); + final FlowFileServiceGrpc.FlowFileServiceBlockingStub blockingStub = blockingStubReference.get(); + final String host = context.getProperty(PROP_SERVICE_HOST).getValue(); + final String port = context.getProperty(PROP_SERVICE_PORT).getValue(); + fileToProcess = session.putAttribute(fileToProcess, SERVICE_HOST, host); + fileToProcess = session.putAttribute(fileToProcess, SERVICE_PORT, port); + FlowFile responseFlowFile = null; + try { + final FlowFileRequest.Builder requestBuilder = FlowFileRequest.newBuilder() + .setId(fileToProcess.getId()) + .putAllAttributes(fileToProcess.getAttributes()); + + // if the processor is configured to send the content, turn the content into bytes + // and add it to the request. + final boolean sendContent = context.getProperty(PROP_SEND_CONTENT).asBoolean(); + if (sendContent) { + try (final InputStream contents = session.read(fileToProcess)) { + requestBuilder.setContent(ByteString.readFrom(contents)); + } + // emit provenance event + session.getProvenanceReporter().send(fileToProcess, getRemote(host, port), true); + } + final FlowFileRequest flowFileRequest = requestBuilder.build(); + logRequest(logger, host, port, flowFileRequest); + + final FlowFileReply flowFileReply = blockingStub.send(flowFileRequest); + logReply(logger, host, port, flowFileReply); + + final FlowFileReply.ResponseCode responseCode = flowFileReply.getResponseCode(); + final String body = flowFileReply.getBody(); + + fileToProcess = session.putAttribute(fileToProcess, RESPONSE_CODE, String.valueOf(responseCode)); + fileToProcess = session.putAttribute(fileToProcess, RESPONSE_BODY, body); + + responseFlowFile = session.create(fileToProcess); + route(fileToProcess, responseFlowFile, session, context, responseCode); + + } catch (final Exception e) { + // penalize or yield + if (fileToProcess != null) { + logger.error("Routing to {}", REL_FAILURE.getName(), e); + fileToProcess = session.penalize(fileToProcess); + fileToProcess = session.putAttribute(fileToProcess, EXCEPTION_CLASS, e.getClass().getName()); + fileToProcess = session.putAttribute(fileToProcess, EXCEPTION_MESSAGE, e.getMessage()); + // transfer original to failure + session.transfer(fileToProcess, REL_FAILURE); + } else { + logger.error("Yielding processor due to exception encountered as a source processor: {}", e); + context.yield(); + } + + // cleanup + try { + if (responseFlowFile != null) { + session.remove(responseFlowFile); + } + } catch (final Exception e1) { + logger.error("Could not cleanup response flowfile", e1); + } + } + } + + /** + * Route the {@link FlowFile} request and response appropriately, depending on the gRPC service + * response code. + * + * @param request the flowfile request + * @param response the flowfile response + * @param session the processor session + * @param context the processor context + * @param responseCode the gRPC service response code + */ + private void route(FlowFile request, FlowFile response, final ProcessSession session, final ProcessContext context, final FlowFileReply.ResponseCode responseCode) { + boolean responseSent = false; + if (context.getProperty(PROP_OUTPUT_RESPONSE_REGARDLESS).asBoolean()) { + session.transfer(response, REL_RESPONSE); + responseSent = true; + } + + switch (responseCode) { + // if the rpc failed, transfer flowfile to no retry relationship and penalize flowfile + + + // if the rpc succeeded, transfer the request and response flowfiles + case SUCCESS: + session.transfer(request, REL_SUCCESS_REQ); + if (!responseSent) { + session.transfer(response, REL_RESPONSE); + } + break; + + // if the gRPC service responded requesting a retry, then penalize the request and + // transfer it to the retry relationship. The flowfile contains attributes detailing this + // rpc request. + case RETRY: + request = session.penalize(request); + session.transfer(request, REL_RETRY); + // if we haven't sent the response by this point, clean it up. + if (!responseSent) { + session.remove(response); + } + break; + + case ERROR: + case UNRECOGNIZED: // unrecognized response code returned from gRPC service + default: + final boolean penalize = context.getProperty(PROP_PENALIZE_NO_RETRY).asBoolean(); + if (penalize) { + request = session.penalize(request); + } + session.transfer(request, REL_NO_RETRY); + // if we haven't sent the response by this point, clean it up. + if (!responseSent) { + session.remove(response); + } + break; + } + } + + private String getRemote(final String host, final String port) { + return host + ":" + port; + } + + private void logRequest(final ComponentLog logger, final String host, final String port, final FlowFileRequest flowFileRequest) { + logger.debug("\nRequest to remote service:\n\t{}\n{}", + getRemote(host, port), flowFileRequest.toString()); + } + + private void logReply(final ComponentLog logger, final String host, final String port, final FlowFileReply flowFileReply) { + logger.debug("\nResponse from remote service:\n\t{}\n{}", + getRemote(host, port), flowFileReply.toString()); + } +} diff --git a/nifi-nar-bundles/nifi-grpc-bundle/nifi-grpc-processors/src/main/java/org/apache/nifi/processors/grpc/ListenGRPC.java b/nifi-nar-bundles/nifi-grpc-bundle/nifi-grpc-processors/src/main/java/org/apache/nifi/processors/grpc/ListenGRPC.java new file mode 100644 index 0000000000..1d486f3fc0 --- /dev/null +++ b/nifi-nar-bundles/nifi-grpc-bundle/nifi-grpc-processors/src/main/java/org/apache/nifi/processors/grpc/ListenGRPC.java @@ -0,0 +1,233 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.grpc; + +import com.google.common.collect.Sets; +import io.grpc.CompressorRegistry; +import io.grpc.DecompressorRegistry; +import io.grpc.Server; +import io.grpc.ServerInterceptors; +import io.grpc.netty.NettyServerBuilder; +import io.netty.handler.ssl.SslContext; +import org.apache.nifi.annotation.behavior.InputRequirement; +import org.apache.nifi.annotation.behavior.WritesAttribute; +import org.apache.nifi.annotation.behavior.WritesAttributes; +import org.apache.nifi.annotation.documentation.CapabilityDescription; +import org.apache.nifi.annotation.documentation.Tags; +import org.apache.nifi.annotation.lifecycle.OnScheduled; +import org.apache.nifi.annotation.lifecycle.OnStopped; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.components.ValidationContext; +import org.apache.nifi.components.ValidationResult; +import org.apache.nifi.logging.ComponentLog; +import org.apache.nifi.processor.AbstractSessionFactoryProcessor; +import org.apache.nifi.processor.DataUnit; +import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.processor.ProcessSessionFactory; +import org.apache.nifi.processor.Relationship; +import org.apache.nifi.processor.exception.ProcessException; +import org.apache.nifi.processor.util.StandardValidators; +import org.apache.nifi.processors.grpc.ssl.SslContextProvider; +import org.apache.nifi.processors.grpc.util.BackpressureChecker; +import org.apache.nifi.ssl.RestrictedSSLContextService; +import org.apache.nifi.ssl.SSLContextService; +import org.apache.nifi.annotation.documentation.DeprecationNotice; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import java.util.regex.Pattern; + +@InputRequirement(InputRequirement.Requirement.INPUT_FORBIDDEN) +@CapabilityDescription("Starts a gRPC server and listens on the given port to transform the incoming messages into FlowFiles." + + " The message format is defined by the standard gRPC protobuf IDL provided by NiFi. gRPC isn't intended to carry large payloads," + + " so this processor should be used only when FlowFile sizes are on the order of megabytes. The default maximum message size is 4MB.") +@Tags({"ingest", "grpc", "rpc", "listen"}) +@WritesAttributes({ + @WritesAttribute(attribute = GRPCAttributeNames.REMOTE_USER_DN, description = "The DN of the user who sent the FlowFile to this NiFi"), + @WritesAttribute(attribute = GRPCAttributeNames.REMOTE_HOST, description = "The IP of the client who sent the FlowFile to this NiFi") +}) +@DeprecationNotice(reason = "No planned alternatives to be offered. Use custom processors instead.") +public class ListenGRPC extends AbstractSessionFactoryProcessor { + + // properties + public static final PropertyDescriptor PROP_SERVICE_PORT = new PropertyDescriptor.Builder() + .name("Local gRPC service port") + .displayName("Local gRPC Service Port") + .description("The local port that the gRPC service will listen on.") + .required(true) + .addValidator(StandardValidators.PORT_VALIDATOR) + .build(); + public static final PropertyDescriptor PROP_USE_SECURE = new PropertyDescriptor.Builder() + .name("Use TLS") + .displayName("Use TLS") + .description("Whether or not to use TLS to receive the contents of the gRPC messages.") + .required(false) + .defaultValue("false") + .allowableValues("true", "false") + .build(); + public static final PropertyDescriptor PROP_SSL_CONTEXT_SERVICE = new PropertyDescriptor.Builder() + .name("SSL Context Service") + .displayName("SSL Context Service") + .description("The SSL Context Service used to provide server certificate information for TLS (https) connections. Keystore must be configured on the service." + + " If truststore is also configured, it will turn on and require client certificate authentication (Mutual TLS).") + .required(false) + .identifiesControllerService(RestrictedSSLContextService.class) + .dependsOn(PROP_USE_SECURE, "true") + .build(); + public static final PropertyDescriptor PROP_FLOW_CONTROL_WINDOW = new PropertyDescriptor.Builder() + .name("Flow Control Window") + .displayName("Flow Control Window") + .description("The initial HTTP/2 flow control window for both new streams and overall connection." + + " Flow-control schemes ensure that streams on the same connection do not destructively interfere with each other." + + " The default is 1MB.") + .defaultValue("1MB") + .required(false) + .addValidator(StandardValidators.DATA_SIZE_VALIDATOR) + .build(); + public static final PropertyDescriptor PROP_MAX_MESSAGE_SIZE = new PropertyDescriptor.Builder() + .name("Max Message Size") + .displayName("Maximum Message Size") + .description("The maximum size of FlowFiles that this processor will allow to be received." + + " The default is 4MB. If FlowFiles exceed this size, you should consider using another transport mechanism" + + " as gRPC isn't designed for heavy payloads.") + .defaultValue("4MB") + .required(false) + .addValidator(StandardValidators.DATA_SIZE_VALIDATOR) + .build(); + public static final PropertyDescriptor PROP_AUTHORIZED_DN_PATTERN = new PropertyDescriptor.Builder() + .name("Authorized DN Pattern") + .displayName("Authorized DN Pattern") + .description("A Regular Expression to apply against the Distinguished Name of incoming connections. If the Pattern does not match the DN, the connection will be refused." + + " The property will only be used if client certificate authentication (Mutual TLS) has been configured on " + PROP_SSL_CONTEXT_SERVICE.getDisplayName() + "," + + " otherwise it will be ignored.") + .required(false) + .defaultValue(".*") + .addValidator(StandardValidators.REGULAR_EXPRESSION_VALIDATOR) + .dependsOn(PROP_USE_SECURE, "true") + .build(); + + public static final List PROPERTIES = Collections.unmodifiableList(Arrays.asList( + PROP_SERVICE_PORT, + PROP_USE_SECURE, + PROP_SSL_CONTEXT_SERVICE, + PROP_AUTHORIZED_DN_PATTERN, + PROP_FLOW_CONTROL_WINDOW, + PROP_MAX_MESSAGE_SIZE + )); + + // relationships + public static final Relationship REL_SUCCESS = new Relationship.Builder() + .name("Success") + .description("The FlowFile was received successfully.") + .build(); + + public static final Set RELATIONSHIPS = Collections.unmodifiableSet(Sets.newHashSet(Arrays.asList( + REL_SUCCESS + ))); + private final AtomicReference sessionFactoryReference = new AtomicReference<>(); + private volatile Server server = null; + + @Override + public Set getRelationships() { + return RELATIONSHIPS; + } + + @Override + protected List getSupportedPropertyDescriptors() { + return PROPERTIES; + } + + @Override + protected Collection customValidate(ValidationContext context) { + List results = new ArrayList<>(super.customValidate(context)); + + final boolean useSecure = context.getProperty(PROP_USE_SECURE).asBoolean(); + final boolean sslContextServiceConfigured = context.getProperty(PROP_SSL_CONTEXT_SERVICE).isSet(); + + if (useSecure && !sslContextServiceConfigured) { + results.add(new ValidationResult.Builder() + .subject(PROP_SSL_CONTEXT_SERVICE.getDisplayName()) + .valid(false) + .explanation(String.format("'%s' must be configured when '%s' is true", PROP_SSL_CONTEXT_SERVICE.getDisplayName(), PROP_USE_SECURE.getDisplayName())) + .build()); + } + + return results; + } + + @OnScheduled + public void startServer(final ProcessContext context) throws Exception { + final ComponentLog logger = getLogger(); + // gather configured properties + final Integer port = context.getProperty(PROP_SERVICE_PORT).asInteger(); + final Boolean useSecure = context.getProperty(PROP_USE_SECURE).asBoolean(); + final int flowControlWindow = context.getProperty(PROP_FLOW_CONTROL_WINDOW).asDataSize(DataUnit.B).intValue(); + final int maxMessageSize = context.getProperty(PROP_MAX_MESSAGE_SIZE).asDataSize(DataUnit.B).intValue(); + final SSLContextService sslContextService = context.getProperty(PROP_SSL_CONTEXT_SERVICE).asControllerService(SSLContextService.class); + final Pattern authorizedDnPattern = Pattern.compile(context.getProperty(PROP_AUTHORIZED_DN_PATTERN).getValue()); + final FlowFileIngestServiceInterceptor callInterceptor = new FlowFileIngestServiceInterceptor(getLogger()); + callInterceptor.enforceDNPattern(authorizedDnPattern); + + final BackpressureChecker backpressureChecker = new BackpressureChecker(context, getRelationships()); + final FlowFileIngestService flowFileIngestService = new FlowFileIngestService(getLogger(), + sessionFactoryReference, + REL_SUCCESS, + backpressureChecker); + final NettyServerBuilder serverBuilder = NettyServerBuilder.forPort(port) + .addService(ServerInterceptors.intercept(flowFileIngestService, callInterceptor)) + // default (de)compressor registries handle both plaintext and gzip compressed messages + .compressorRegistry(CompressorRegistry.getDefaultInstance()) + .decompressorRegistry(DecompressorRegistry.getDefaultInstance()) + .flowControlWindow(flowControlWindow) + .maxInboundMessageSize(maxMessageSize); + + if (useSecure) { + final SslContext serverSslContext = SslContextProvider.getSslContext(sslContextService, false); + serverBuilder.sslContext(serverSslContext); + } + logger.info("Starting gRPC server on port: {}", port.toString()); + this.server = serverBuilder.build().start(); + } + + public int getListeningPort() { + return server == null ? 0 : server.getPort(); + } + + @OnStopped + public void stopServer(final ProcessContext context) { + if (this.server != null) { + try { + this.server.shutdown().awaitTermination(5, TimeUnit.SECONDS); + } catch (InterruptedException e) { + getLogger().warn("Unable to cleanly shutdown embedded gRPC server", e); + this.server = null; + } + } + } + + @Override + public void onTrigger(ProcessContext context, ProcessSessionFactory sessionFactory) throws ProcessException { + sessionFactoryReference.compareAndSet(null, sessionFactory); + context.yield(); + } +} diff --git a/nifi-nar-bundles/nifi-grpc-bundle/nifi-grpc-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor b/nifi-nar-bundles/nifi-grpc-bundle/nifi-grpc-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor new file mode 100644 index 0000000000..10609a739c --- /dev/null +++ b/nifi-nar-bundles/nifi-grpc-bundle/nifi-grpc-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +org.apache.nifi.processors.grpc.InvokeGRPC +org.apache.nifi.processors.grpc.ListenGRPC diff --git a/nifi-nar-bundles/nifi-grpc-bundle/nifi-grpc-processors/src/test/java/org/apache/nifi/processors/grpc/ITListenGRPC.java b/nifi-nar-bundles/nifi-grpc-bundle/nifi-grpc-processors/src/test/java/org/apache/nifi/processors/grpc/ITListenGRPC.java new file mode 100644 index 0000000000..ac2530de6d --- /dev/null +++ b/nifi-nar-bundles/nifi-grpc-bundle/nifi-grpc-processors/src/test/java/org/apache/nifi/processors/grpc/ITListenGRPC.java @@ -0,0 +1,170 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.grpc; + +import com.google.common.collect.Sets; +import com.google.protobuf.ByteString; +import io.grpc.ManagedChannel; +import org.apache.nifi.flowfile.attributes.CoreAttributes; +import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.processor.ProcessSessionFactory; +import org.apache.nifi.util.MockFlowFile; +import org.apache.nifi.util.TestRunner; +import org.apache.nifi.util.TestRunners; +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.core.StringContains.containsString; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; + +public class ITListenGRPC { + private static final String HOST = "localhost"; + private static final String SOURCE_SYSTEM_UUID = "FAKE_UUID"; + + @Test + public void testSuccessfulRoundTrip() throws Exception { + final int randPort = TestGRPCClient.randomPort(); + final ManagedChannel channel = TestGRPCClient.buildChannel(HOST, randPort); + final FlowFileServiceGrpc.FlowFileServiceBlockingStub stub = FlowFileServiceGrpc.newBlockingStub(channel); + + final ListenGRPC listenGRPC = new ListenGRPC(); + final TestRunner runner = TestRunners.newTestRunner(listenGRPC); + runner.setProperty(ListenGRPC.PROP_SERVICE_PORT, String.valueOf(randPort)); + + final ProcessContext processContext = runner.getProcessContext(); + final ProcessSessionFactory processSessionFactory = runner.getProcessSessionFactory(); + + try { + // start the server. The order of the following statements shouldn't matter, because the + // startServer() method waits for a processSessionFactory to be available to it. + listenGRPC.startServer(processContext); + listenGRPC.onTrigger(processContext, processSessionFactory); + + final FlowFileRequest ingestFile = FlowFileRequest.newBuilder() + .putAttributes("FOO", "BAR") + .putAttributes(CoreAttributes.UUID.key(), SOURCE_SYSTEM_UUID) + .setContent(ByteString.copyFrom("content".getBytes())) + .build(); + final FlowFileReply reply = stub.send(ingestFile); + assertThat(reply.getResponseCode(), equalTo(FlowFileReply.ResponseCode.SUCCESS)); + assertThat(reply.getBody(), equalTo("FlowFile successfully received.")); + + runner.assertTransferCount(ListenGRPC.REL_SUCCESS, 1); + final List successFiles = runner.getFlowFilesForRelationship(ListenGRPC.REL_SUCCESS); + assertThat(successFiles.size(), equalTo(1)); + final MockFlowFile mockFlowFile = successFiles.get(0); + assertThat(mockFlowFile.getAttribute("FOO"), equalTo("BAR")); + assertThat(mockFlowFile.getAttribute(GRPCAttributeNames.REMOTE_HOST), equalTo("127.0.0.1")); + assertThat(mockFlowFile.getAttribute(GRPCAttributeNames.REMOTE_USER_DN), equalTo(FlowFileIngestServiceInterceptor.DEFAULT_FOUND_SUBJECT)); + + } finally { + // stop the server + listenGRPC.stopServer(processContext); + channel.shutdown(); + } + } + + @Test + public void testOutOfSpaceRoundTrip() throws Exception { + final int randPort = TestGRPCClient.randomPort(); + final ManagedChannel channel = TestGRPCClient.buildChannel(HOST, randPort); + final FlowFileServiceGrpc.FlowFileServiceBlockingStub stub = FlowFileServiceGrpc.newBlockingStub(channel); + + final ListenGRPC listenGRPC = new ListenGRPC(); + final TestRunner runner = TestRunners.newTestRunner(listenGRPC); + runner.setProperty(ListenGRPC.PROP_SERVICE_PORT, String.valueOf(randPort)); + + final ProcessContext processContext = spy(runner.getProcessContext()); + // force the context to return that space isn't available, prompting an error message to be returned. + when(processContext.getAvailableRelationships()).thenReturn(Sets.newHashSet()); + final ProcessSessionFactory processSessionFactory = runner.getProcessSessionFactory(); + + try { + // start the server. The order of the following statements shouldn't matter, because the + // startServer() method waits for a processSessionFactory to be available to it. + listenGRPC.startServer(processContext); + listenGRPC.onTrigger(processContext, processSessionFactory); + + final FlowFileRequest ingestFile = FlowFileRequest.newBuilder() + .putAttributes("FOO", "BAR") + .setContent(ByteString.copyFrom("content".getBytes())) + .build(); + final FlowFileReply reply = stub.send(ingestFile); + assertThat(reply.getResponseCode(), equalTo(FlowFileReply.ResponseCode.ERROR)); + assertThat(reply.getBody(), containsString("but no space available; Indicating Service Unavailable")); + + runner.assertTransferCount(ListenGRPC.REL_SUCCESS, 0); + } finally { + // stop the server + listenGRPC.stopServer(processContext); + channel.shutdown(); + } + } + + @Test + public void testExceedMaxMessageSize() { + assertThrows(io.grpc.StatusRuntimeException.class, () -> { + final int randPort = TestGRPCClient.randomPort(); + final ManagedChannel channel = TestGRPCClient.buildChannel(HOST, randPort); + final FlowFileServiceGrpc.FlowFileServiceBlockingStub stub = FlowFileServiceGrpc.newBlockingStub(channel); + + final ListenGRPC listenGRPC = new ListenGRPC(); + final TestRunner runner = TestRunners.newTestRunner(listenGRPC); + runner.setProperty(ListenGRPC.PROP_SERVICE_PORT, String.valueOf(randPort)); + // set max message size to 1 byte to force exception to be thrown. + runner.setProperty(ListenGRPC.PROP_MAX_MESSAGE_SIZE, "1B"); + + final ProcessContext processContext = runner.getProcessContext(); + final ProcessSessionFactory processSessionFactory = runner.getProcessSessionFactory(); + + try { + // start the server. The order of the following statements shouldn't matter, because the + // startServer() method waits for a processSessionFactory to be available to it. + listenGRPC.startServer(processContext); + listenGRPC.onTrigger(processContext, processSessionFactory); + + final FlowFileRequest ingestFile = FlowFileRequest.newBuilder() + .putAttributes("FOO", "BAR") + .putAttributes(CoreAttributes.UUID.key(), SOURCE_SYSTEM_UUID) + .setContent(ByteString.copyFrom("content".getBytes())) + .build(); + // this should throw a runtime exception + final FlowFileReply reply = stub.send(ingestFile); + assertThat(reply.getResponseCode(), equalTo(FlowFileReply.ResponseCode.SUCCESS)); + assertThat(reply.getBody(), equalTo("FlowFile successfully received.")); + + runner.assertTransferCount(ListenGRPC.REL_SUCCESS, 1); + final List successFiles = runner.getFlowFilesForRelationship(ListenGRPC.REL_SUCCESS); + assertThat(successFiles.size(), equalTo(1)); + final MockFlowFile mockFlowFile = successFiles.get(0); + assertThat(mockFlowFile.getAttribute("FOO"), equalTo("BAR")); + assertThat(mockFlowFile.getAttribute(GRPCAttributeNames.REMOTE_HOST), equalTo("127.0.0.1")); + assertThat(mockFlowFile.getAttribute(GRPCAttributeNames.REMOTE_USER_DN), equalTo(FlowFileIngestServiceInterceptor.DEFAULT_FOUND_SUBJECT)); + + } finally { + // stop the server + listenGRPC.stopServer(processContext); + channel.shutdown(); + } + }); + } +} \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-grpc-bundle/nifi-grpc-processors/src/test/java/org/apache/nifi/processors/grpc/TestGRPCClient.java b/nifi-nar-bundles/nifi-grpc-bundle/nifi-grpc-processors/src/test/java/org/apache/nifi/processors/grpc/TestGRPCClient.java new file mode 100644 index 0000000000..dbddf79cea --- /dev/null +++ b/nifi-nar-bundles/nifi-grpc-bundle/nifi-grpc-processors/src/test/java/org/apache/nifi/processors/grpc/TestGRPCClient.java @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.grpc; + +import org.apache.nifi.ssl.StandardSSLContextService; + +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.security.KeyStore; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; +import java.security.UnrecoverableKeyException; +import java.security.cert.CertificateException; +import java.util.Map; +import java.util.concurrent.ThreadLocalRandom; + +import javax.net.ssl.KeyManagerFactory; +import javax.net.ssl.TrustManagerFactory; + +import io.grpc.CompressorRegistry; +import io.grpc.DecompressorRegistry; +import io.grpc.ManagedChannel; +import io.grpc.netty.GrpcSslContexts; +import io.grpc.netty.NettyChannelBuilder; +import io.netty.handler.ssl.ClientAuth; +import io.netty.handler.ssl.SslContextBuilder; + +import static org.apache.nifi.processors.grpc.TestGRPCServer.NEED_CLIENT_AUTH; + +/** + * Generic gRPC channel builder for use in unit testing. Consumers should use the channel built here + * to construct the desired stubs to communicate with a gRPC service. + */ +public class TestGRPCClient { + // Used to represent the ephemeral port range. + private static final int PORT_START = 49152; + private static final int PORT_END = 65535; + + /** + * Can be used by clients to grab a random port in a range of ports + * + * @return a port to use for client/server comms + */ + public static int randomPort() { + // add 1 because upper bound is exclusive + return ThreadLocalRandom.current().nextInt(PORT_START, PORT_END + 1); + } + + /** + * Build a channel with the given host and port and optional ssl properties. + * + * @param host the host to establish a connection with + * @param port the port on which to communicate with the host + * @return a constructed channel + */ + public static ManagedChannel buildChannel(final String host, final int port) + throws NoSuchAlgorithmException, KeyStoreException, IOException, CertificateException, UnrecoverableKeyException { + return buildChannel(host, port, null); + } + + /** + * Build a channel with the given host and port and optional ssl properties. + * + * @param host the host to establish a connection with + * @param port the port on which to communicate with the host + * @param sslProperties the properties by which to establish an ssl connection + * @return a constructed channel + */ + public static ManagedChannel buildChannel(final String host, final int port, final Map sslProperties) + throws NoSuchAlgorithmException, KeyStoreException, IOException, CertificateException, UnrecoverableKeyException { + final NettyChannelBuilder channelBuilder = NettyChannelBuilder.forAddress(host, port) + .directExecutor() + .compressorRegistry(CompressorRegistry.getDefaultInstance()) + .decompressorRegistry(DecompressorRegistry.getDefaultInstance()) + .userAgent("testAgent"); + + if (sslProperties != null) { + final SslContextBuilder sslContextBuilder = SslContextBuilder.forClient(); + + if(sslProperties.get(StandardSSLContextService.KEYSTORE.getName()) != null) { + final KeyManagerFactory keyManager = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); + final KeyStore keyStore = KeyStore.getInstance(sslProperties.get(StandardSSLContextService.KEYSTORE_TYPE.getName())); + final String keyStoreFile = sslProperties.get(StandardSSLContextService.KEYSTORE.getName()); + final String keyStorePassword = sslProperties.get(StandardSSLContextService.KEYSTORE_PASSWORD.getName()); + try (final InputStream is = new FileInputStream(keyStoreFile)) { + keyStore.load(is, keyStorePassword.toCharArray()); + } + keyManager.init(keyStore, keyStorePassword.toCharArray()); + sslContextBuilder.keyManager(keyManager); + } + + if (sslProperties.get(StandardSSLContextService.TRUSTSTORE.getName()) != null) { + final TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); + final KeyStore trustStore = KeyStore.getInstance(sslProperties.get(StandardSSLContextService.TRUSTSTORE_TYPE.getName())); + final String trustStoreFile = sslProperties.get(StandardSSLContextService.TRUSTSTORE.getName()); + final String trustStorePassword = sslProperties.get(StandardSSLContextService.TRUSTSTORE_PASSWORD.getName()); + try (final InputStream is = new FileInputStream(trustStoreFile)) { + trustStore.load(is, trustStorePassword.toCharArray()); + } + trustManagerFactory.init(trustStore); + sslContextBuilder.trustManager(trustManagerFactory); + } + + final String clientAuth = sslProperties.get(NEED_CLIENT_AUTH); + if (clientAuth == null) { + sslContextBuilder.clientAuth(ClientAuth.REQUIRE); + } else { + sslContextBuilder.clientAuth(ClientAuth.valueOf(clientAuth)); + } + GrpcSslContexts.configure(sslContextBuilder); + channelBuilder.sslContext(sslContextBuilder.build()); + } else { + channelBuilder.usePlaintext(); + } + return channelBuilder.build(); + } +} diff --git a/nifi-nar-bundles/nifi-grpc-bundle/nifi-grpc-processors/src/test/java/org/apache/nifi/processors/grpc/TestGRPCServer.java b/nifi-nar-bundles/nifi-grpc-bundle/nifi-grpc-processors/src/test/java/org/apache/nifi/processors/grpc/TestGRPCServer.java new file mode 100644 index 0000000000..b885853064 --- /dev/null +++ b/nifi-nar-bundles/nifi-grpc-bundle/nifi-grpc-processors/src/test/java/org/apache/nifi/processors/grpc/TestGRPCServer.java @@ -0,0 +1,166 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.grpc; + +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.ServerSocket; +import java.security.KeyStore; +import java.util.Map; + +import javax.net.ssl.KeyManagerFactory; +import javax.net.ssl.TrustManagerFactory; + +import org.apache.nifi.ssl.StandardSSLContextService; + +import io.grpc.BindableService; +import io.grpc.CompressorRegistry; +import io.grpc.DecompressorRegistry; +import io.grpc.Server; +import io.grpc.netty.GrpcSslContexts; +import io.grpc.netty.NettyServerBuilder; +import io.netty.handler.ssl.ClientAuth; +import io.netty.handler.ssl.SslContextBuilder; + +/** + * Generic gRPC test server to assist with unit tests that require a server to be present. + * + * @param the gRPC service implementation + */ +public class TestGRPCServer { + public static final String HOST = "localhost"; + public static final String NEED_CLIENT_AUTH = "needClientAuth"; + private final Class clazz; + private final Map sslProperties; + private Server server; + + /** + * Create a gRPC server + * + * @param clazz the gRPC service implementation + */ + public TestGRPCServer(final Class clazz) { + this(clazz, null); + } + + + /** + * Create a gRPC server + * + * @param clazz the gRPC service implementation + * @param sslProperties the keystore and truststore properties for SSL communications + */ + public TestGRPCServer(final Class clazz, final Map sslProperties) { + this.clazz = clazz; + this.sslProperties = sslProperties; + } + + /** + * Can be used by clients to grab a random port in a range of ports + * + * @return a port to use for client/server comms + */ + public static int randomPort() throws IOException { + ServerSocket socket = new ServerSocket(0); + socket.setReuseAddress(true); + final int port = socket.getLocalPort(); + socket.close(); + return port; + } + + /** + * Starts the gRPC server @localhost:port. + */ + public int start(final int port) throws Exception { + final NettyServerBuilder nettyServerBuilder = NettyServerBuilder + .forPort(port) + .directExecutor() + .addService(clazz.newInstance()) + .compressorRegistry(CompressorRegistry.getDefaultInstance()) + .decompressorRegistry(DecompressorRegistry.getDefaultInstance()); + + if (this.sslProperties != null) { + if (sslProperties.get(StandardSSLContextService.KEYSTORE.getName()) == null) { + throw new RuntimeException("You must configure a keystore in order to use SSL with gRPC."); + } + + final KeyManagerFactory keyManager = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); + final KeyStore keyStore = KeyStore.getInstance(sslProperties.get(StandardSSLContextService.KEYSTORE_TYPE.getName())); + final String keyStoreFile = sslProperties.get(StandardSSLContextService.KEYSTORE.getName()); + final String keyStorePassword = sslProperties.get(StandardSSLContextService.KEYSTORE_PASSWORD.getName()); + try (final InputStream is = new FileInputStream(keyStoreFile)) { + keyStore.load(is, keyStorePassword.toCharArray()); + } + keyManager.init(keyStore, keyStorePassword.toCharArray()); + final SslContextBuilder sslContextBuilder = SslContextBuilder.forServer(keyManager); + + if (sslProperties.get(StandardSSLContextService.TRUSTSTORE.getName()) != null) { + final TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); + final KeyStore trustStore = KeyStore.getInstance(sslProperties.get(StandardSSLContextService.TRUSTSTORE_TYPE.getName())); + final String trustStoreFile = sslProperties.get(StandardSSLContextService.TRUSTSTORE.getName()); + final String trustStorePassword = sslProperties.get(StandardSSLContextService.TRUSTSTORE_PASSWORD.getName()); + try (final InputStream is = new FileInputStream(trustStoreFile)) { + trustStore.load(is, trustStorePassword.toCharArray()); + } + trustManagerFactory.init(trustStore); + sslContextBuilder.trustManager(trustManagerFactory); + } + + final String clientAuth = sslProperties.get(NEED_CLIENT_AUTH); + if (clientAuth == null) { + sslContextBuilder.clientAuth(ClientAuth.REQUIRE); + } else { + sslContextBuilder.clientAuth(ClientAuth.valueOf(clientAuth)); + } + GrpcSslContexts.configure(sslContextBuilder); + nettyServerBuilder.sslContext(sslContextBuilder.build()); + } + + server = nettyServerBuilder.build().start(); + final int actualPort = server.getPort(); + + Runtime.getRuntime().addShutdownHook(new Thread() { + @Override + public void run() { + // Use stderr here since the logger may have been reset by its JVM shutdown hook. + System.err.println("*** shutting down gRPC server since JVM is shutting down"); + TestGRPCServer.this.stop(); + System.err.println("*** server shut down"); + } + }); + return actualPort; + } + + /** + * Stop the server. + */ + void stop() { + if (server != null) { + server.shutdown(); + } + } + + /** + * Await termination on the main thread since the grpc library uses daemon threads. + */ + public void blockUntilShutdown() throws InterruptedException { + if (server != null) { + server.awaitTermination(); + } + } +} diff --git a/nifi-nar-bundles/nifi-grpc-bundle/nifi-grpc-processors/src/test/java/org/apache/nifi/processors/grpc/TestInvokeGRPC.java b/nifi-nar-bundles/nifi-grpc-bundle/nifi-grpc-processors/src/test/java/org/apache/nifi/processors/grpc/TestInvokeGRPC.java new file mode 100644 index 0000000000..be05a2ca2c --- /dev/null +++ b/nifi-nar-bundles/nifi-grpc-bundle/nifi-grpc-processors/src/test/java/org/apache/nifi/processors/grpc/TestInvokeGRPC.java @@ -0,0 +1,422 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.grpc; + +import io.grpc.stub.StreamObserver; +import org.apache.nifi.util.MockFlowFile; +import org.apache.nifi.util.TestRunner; +import org.apache.nifi.util.TestRunners; +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.MatcherAssert.assertThat; + +public class TestInvokeGRPC { + // ids placed on flowfiles and used to dictate response codes in the DummyFlowFileService below + private static final long ERROR = 500; + private static final long SUCCESS = 501; + private static final long RETRY = 502; + + @Test + public void testSuccess() throws Exception { + final TestGRPCServer server = new TestGRPCServer<>(DummyFlowFileService.class); + + try { + final int port = server.start(0); + final TestRunner runner = TestRunners.newTestRunner(InvokeGRPC.class); + runner.setProperty(InvokeGRPC.PROP_SERVICE_HOST, TestGRPCServer.HOST); + runner.setProperty(InvokeGRPC.PROP_SERVICE_PORT, String.valueOf(port)); + + final MockFlowFile mockFlowFile = new MockFlowFile(SUCCESS); + runner.enqueue(mockFlowFile); + runner.run(); + runner.assertTransferCount(InvokeGRPC.REL_RESPONSE, 1); + runner.assertTransferCount(InvokeGRPC.REL_SUCCESS_REQ, 1); + runner.assertTransferCount(InvokeGRPC.REL_RETRY, 0); + runner.assertTransferCount(InvokeGRPC.REL_NO_RETRY, 0); + runner.assertTransferCount(InvokeGRPC.REL_FAILURE, 0); + + final List responseFiles = runner.getFlowFilesForRelationship(InvokeGRPC.REL_RESPONSE); + assertThat(responseFiles.size(), equalTo(1)); + final MockFlowFile response = responseFiles.get(0); + response.assertAttributeEquals(InvokeGRPC.RESPONSE_CODE, String.valueOf(FlowFileReply.ResponseCode.SUCCESS)); + response.assertAttributeEquals(InvokeGRPC.RESPONSE_BODY, "success"); + response.assertAttributeEquals(InvokeGRPC.SERVICE_HOST, TestGRPCServer.HOST); + response.assertAttributeEquals(InvokeGRPC.SERVICE_PORT, String.valueOf(port)); + + final List successFiles = runner.getFlowFilesForRelationship(InvokeGRPC.REL_SUCCESS_REQ); + assertThat(successFiles.size(), equalTo(1)); + final MockFlowFile successFile = successFiles.get(0); + successFile.assertAttributeEquals(InvokeGRPC.RESPONSE_CODE, String.valueOf(FlowFileReply.ResponseCode.SUCCESS)); + successFile.assertAttributeEquals(InvokeGRPC.RESPONSE_BODY, "success"); + successFile.assertAttributeEquals(InvokeGRPC.SERVICE_HOST, TestGRPCServer.HOST); + successFile.assertAttributeEquals(InvokeGRPC.SERVICE_PORT, String.valueOf(port)); + } finally { + server.stop(); + } + } + + @Test + public void testSuccessWithFlowFileContent() throws Exception { + final TestGRPCServer server = new TestGRPCServer<>(DummyFlowFileService.class); + + try { + final int port = server.start(0); + + final TestRunner runner = TestRunners.newTestRunner(InvokeGRPC.class); + runner.setProperty(InvokeGRPC.PROP_SERVICE_HOST, TestGRPCServer.HOST); + runner.setProperty(InvokeGRPC.PROP_SERVICE_PORT, String.valueOf(port)); + + runner.enqueue("content"); + runner.run(); + runner.assertTransferCount(InvokeGRPC.REL_RESPONSE, 1); + runner.assertTransferCount(InvokeGRPC.REL_SUCCESS_REQ, 1); + runner.assertTransferCount(InvokeGRPC.REL_RETRY, 0); + runner.assertTransferCount(InvokeGRPC.REL_NO_RETRY, 0); + runner.assertTransferCount(InvokeGRPC.REL_FAILURE, 0); + + final List responseFiles = runner.getFlowFilesForRelationship(InvokeGRPC.REL_RESPONSE); + assertThat(responseFiles.size(), equalTo(1)); + final MockFlowFile response = responseFiles.get(0); + response.assertAttributeEquals(InvokeGRPC.RESPONSE_CODE, String.valueOf(FlowFileReply.ResponseCode.SUCCESS)); + response.assertAttributeEquals(InvokeGRPC.RESPONSE_BODY, "content"); + response.assertAttributeEquals(InvokeGRPC.SERVICE_HOST, TestGRPCServer.HOST); + response.assertAttributeEquals(InvokeGRPC.SERVICE_PORT, String.valueOf(port)); + + final List successFiles = runner.getFlowFilesForRelationship(InvokeGRPC.REL_SUCCESS_REQ); + assertThat(successFiles.size(), equalTo(1)); + final MockFlowFile successFile = successFiles.get(0); + successFile.assertAttributeEquals(InvokeGRPC.RESPONSE_CODE, String.valueOf(FlowFileReply.ResponseCode.SUCCESS)); + successFile.assertAttributeEquals(InvokeGRPC.RESPONSE_BODY, "content"); + successFile.assertAttributeEquals(InvokeGRPC.SERVICE_HOST, TestGRPCServer.HOST); + successFile.assertAttributeEquals(InvokeGRPC.SERVICE_PORT, String.valueOf(port)); + } finally { + server.stop(); + } + } + + @Test + public void testSuccessAlwaysOutputResponse() throws Exception { + final TestGRPCServer server = new TestGRPCServer<>(DummyFlowFileService.class); + + try { + final int port = server.start(0); + + final TestRunner runner = TestRunners.newTestRunner(InvokeGRPC.class); + runner.setProperty(InvokeGRPC.PROP_SERVICE_HOST, TestGRPCServer.HOST); + runner.setProperty(InvokeGRPC.PROP_SERVICE_PORT, String.valueOf(port)); + runner.setProperty(InvokeGRPC.PROP_OUTPUT_RESPONSE_REGARDLESS, "true"); + + final MockFlowFile mockFlowFile = new MockFlowFile(SUCCESS); + runner.enqueue(mockFlowFile); + runner.run(); + runner.assertTransferCount(InvokeGRPC.REL_RESPONSE, 1); + runner.assertTransferCount(InvokeGRPC.REL_SUCCESS_REQ, 1); + runner.assertTransferCount(InvokeGRPC.REL_RETRY, 0); + runner.assertTransferCount(InvokeGRPC.REL_NO_RETRY, 0); + runner.assertTransferCount(InvokeGRPC.REL_FAILURE, 0); + + final List responseFiles = runner.getFlowFilesForRelationship(InvokeGRPC.REL_RESPONSE); + assertThat(responseFiles.size(), equalTo(1)); + final MockFlowFile response = responseFiles.get(0); + response.assertAttributeEquals(InvokeGRPC.RESPONSE_CODE, String.valueOf(FlowFileReply.ResponseCode.SUCCESS)); + response.assertAttributeEquals(InvokeGRPC.RESPONSE_BODY, "success"); + response.assertAttributeEquals(InvokeGRPC.SERVICE_HOST, TestGRPCServer.HOST); + response.assertAttributeEquals(InvokeGRPC.SERVICE_PORT, String.valueOf(port)); + + final List successFiles = runner.getFlowFilesForRelationship(InvokeGRPC.REL_SUCCESS_REQ); + assertThat(successFiles.size(), equalTo(1)); + final MockFlowFile successFile = successFiles.get(0); + successFile.assertAttributeEquals(InvokeGRPC.RESPONSE_CODE, String.valueOf(FlowFileReply.ResponseCode.SUCCESS)); + successFile.assertAttributeEquals(InvokeGRPC.RESPONSE_BODY, "success"); + successFile.assertAttributeEquals(InvokeGRPC.SERVICE_HOST, TestGRPCServer.HOST); + successFile.assertAttributeEquals(InvokeGRPC.SERVICE_PORT, String.valueOf(port)); + } finally { + server.stop(); + } + } + + @Test + public void testExceedMaxMessageSize() throws Exception { + final TestGRPCServer server = new TestGRPCServer<>(DummyFlowFileService.class); + + try { + final int port = server.start(0); + final TestRunner runner = TestRunners.newTestRunner(InvokeGRPC.class); + runner.setProperty(InvokeGRPC.PROP_SERVICE_HOST, TestGRPCServer.HOST); + runner.setProperty(InvokeGRPC.PROP_SERVICE_PORT, String.valueOf(port)); + // set max message size to 1B to force error + runner.setProperty(InvokeGRPC.PROP_MAX_MESSAGE_SIZE, "1B"); + + final MockFlowFile mockFlowFile = new MockFlowFile(SUCCESS); + runner.enqueue(mockFlowFile); + runner.run(); + runner.assertTransferCount(InvokeGRPC.REL_RESPONSE, 0); + runner.assertTransferCount(InvokeGRPC.REL_SUCCESS_REQ, 0); + runner.assertTransferCount(InvokeGRPC.REL_RETRY, 0); + runner.assertTransferCount(InvokeGRPC.REL_NO_RETRY, 0); + runner.assertTransferCount(InvokeGRPC.REL_FAILURE, 1); + + final List responseFiles = runner.getFlowFilesForRelationship(InvokeGRPC.REL_FAILURE); + assertThat(responseFiles.size(), equalTo(1)); + final MockFlowFile response = responseFiles.get(0); + response.assertAttributeEquals(InvokeGRPC.SERVICE_HOST, TestGRPCServer.HOST); + response.assertAttributeEquals(InvokeGRPC.SERVICE_PORT, String.valueOf(port)); + // an exception should be thrown indicating that the max message size was exceeded. + response.assertAttributeEquals(InvokeGRPC.EXCEPTION_CLASS, "io.grpc.StatusRuntimeException"); + } finally { + server.stop(); + } + } + + @Test + public void testRetry() throws Exception { + final TestGRPCServer server = new TestGRPCServer<>(DummyFlowFileService.class); + + try { + final int port = server.start(0); + + final TestRunner runner = TestRunners.newTestRunner(InvokeGRPC.class); + runner.setProperty(InvokeGRPC.PROP_SERVICE_HOST, TestGRPCServer.HOST); + runner.setProperty(InvokeGRPC.PROP_SERVICE_PORT, String.valueOf(port)); + + final MockFlowFile mockFlowFile = new MockFlowFile(RETRY); + runner.enqueue(mockFlowFile); + runner.run(); + runner.assertTransferCount(InvokeGRPC.REL_RESPONSE, 0); + runner.assertTransferCount(InvokeGRPC.REL_SUCCESS_REQ, 0); + runner.assertTransferCount(InvokeGRPC.REL_RETRY, 1); + runner.assertTransferCount(InvokeGRPC.REL_NO_RETRY, 0); + runner.assertTransferCount(InvokeGRPC.REL_FAILURE, 0); + runner.assertPenalizeCount(1); + + final List responseFiles = runner.getFlowFilesForRelationship(InvokeGRPC.REL_RETRY); + assertThat(responseFiles.size(), equalTo(1)); + final MockFlowFile response = responseFiles.get(0); + response.assertAttributeEquals(InvokeGRPC.RESPONSE_CODE, String.valueOf(FlowFileReply.ResponseCode.RETRY)); + response.assertAttributeEquals(InvokeGRPC.RESPONSE_BODY, "retry"); + response.assertAttributeEquals(InvokeGRPC.SERVICE_HOST, TestGRPCServer.HOST); + response.assertAttributeEquals(InvokeGRPC.SERVICE_PORT, String.valueOf(port)); + } finally { + server.stop(); + } + } + + @Test + public void testRetryAlwaysOutputResponse() throws Exception { + final TestGRPCServer server = new TestGRPCServer<>(DummyFlowFileService.class); + + try { + final int port = server.start(0); + final TestRunner runner = TestRunners.newTestRunner(InvokeGRPC.class); + runner.setProperty(InvokeGRPC.PROP_SERVICE_HOST, TestGRPCServer.HOST); + runner.setProperty(InvokeGRPC.PROP_SERVICE_PORT, String.valueOf(port)); + runner.setProperty(InvokeGRPC.PROP_OUTPUT_RESPONSE_REGARDLESS, "true"); + + final MockFlowFile mockFlowFile = new MockFlowFile(RETRY); + runner.enqueue(mockFlowFile); + runner.run(); + runner.assertTransferCount(InvokeGRPC.REL_RESPONSE, 1); + runner.assertTransferCount(InvokeGRPC.REL_SUCCESS_REQ, 0); + runner.assertTransferCount(InvokeGRPC.REL_RETRY, 1); + runner.assertTransferCount(InvokeGRPC.REL_NO_RETRY, 0); + runner.assertTransferCount(InvokeGRPC.REL_FAILURE, 0); + runner.assertPenalizeCount(1); + + final List retryFiles = runner.getFlowFilesForRelationship(InvokeGRPC.REL_RETRY); + assertThat(retryFiles.size(), equalTo(1)); + final MockFlowFile retry = retryFiles.get(0); + retry.assertAttributeEquals(InvokeGRPC.RESPONSE_CODE, String.valueOf(FlowFileReply.ResponseCode.RETRY)); + retry.assertAttributeEquals(InvokeGRPC.RESPONSE_BODY, "retry"); + retry.assertAttributeEquals(InvokeGRPC.SERVICE_HOST, TestGRPCServer.HOST); + retry.assertAttributeEquals(InvokeGRPC.SERVICE_PORT, String.valueOf(port)); + + final List responseFiles = runner.getFlowFilesForRelationship(InvokeGRPC.REL_RESPONSE); + assertThat(responseFiles.size(), equalTo(1)); + final MockFlowFile response = responseFiles.get(0); + response.assertAttributeEquals(InvokeGRPC.RESPONSE_CODE, String.valueOf(FlowFileReply.ResponseCode.RETRY)); + response.assertAttributeEquals(InvokeGRPC.RESPONSE_BODY, "retry"); + response.assertAttributeEquals(InvokeGRPC.SERVICE_HOST, TestGRPCServer.HOST); + response.assertAttributeEquals(InvokeGRPC.SERVICE_PORT, String.valueOf(port)); + } finally { + server.stop(); + } + } + + @Test + public void testNoRetryOnError() throws Exception { + final TestGRPCServer server = new TestGRPCServer<>(DummyFlowFileService.class); + + try { + final int port = server.start(0); + + final TestRunner runner = TestRunners.newTestRunner(InvokeGRPC.class); + runner.setProperty(InvokeGRPC.PROP_SERVICE_HOST, TestGRPCServer.HOST); + runner.setProperty(InvokeGRPC.PROP_SERVICE_PORT, String.valueOf(port)); + + final MockFlowFile mockFlowFile = new MockFlowFile(ERROR); + runner.enqueue(mockFlowFile); + runner.run(); + runner.assertTransferCount(InvokeGRPC.REL_RESPONSE, 0); + runner.assertTransferCount(InvokeGRPC.REL_SUCCESS_REQ, 0); + runner.assertTransferCount(InvokeGRPC.REL_RETRY, 0); + runner.assertTransferCount(InvokeGRPC.REL_NO_RETRY, 1); + runner.assertTransferCount(InvokeGRPC.REL_FAILURE, 0); + + final List responseFiles = runner.getFlowFilesForRelationship(InvokeGRPC.REL_NO_RETRY); + assertThat(responseFiles.size(), equalTo(1)); + final MockFlowFile response = responseFiles.get(0); + response.assertAttributeEquals(InvokeGRPC.RESPONSE_CODE, String.valueOf(FlowFileReply.ResponseCode.ERROR)); + response.assertAttributeEquals(InvokeGRPC.RESPONSE_BODY, "error"); + response.assertAttributeEquals(InvokeGRPC.SERVICE_HOST, TestGRPCServer.HOST); + response.assertAttributeEquals(InvokeGRPC.SERVICE_PORT, String.valueOf(port)); + } finally { + server.stop(); + } + } + + @Test + public void testNoRetryOnErrorAlwaysOutputResponseAndPenalize() throws Exception { + final TestGRPCServer server = new TestGRPCServer<>(DummyFlowFileService.class); + + try { + final int port = server.start(0); + + final TestRunner runner = TestRunners.newTestRunner(InvokeGRPC.class); + runner.setProperty(InvokeGRPC.PROP_SERVICE_HOST, TestGRPCServer.HOST); + runner.setProperty(InvokeGRPC.PROP_SERVICE_PORT, String.valueOf(port)); + runner.setProperty(InvokeGRPC.PROP_OUTPUT_RESPONSE_REGARDLESS, "true"); + runner.setProperty(InvokeGRPC.PROP_PENALIZE_NO_RETRY, "true"); + + final MockFlowFile mockFlowFile = new MockFlowFile(ERROR); + runner.enqueue(mockFlowFile); + runner.run(); + runner.assertTransferCount(InvokeGRPC.REL_RESPONSE, 1); + runner.assertTransferCount(InvokeGRPC.REL_SUCCESS_REQ, 0); + runner.assertTransferCount(InvokeGRPC.REL_RETRY, 0); + runner.assertTransferCount(InvokeGRPC.REL_NO_RETRY, 1); + runner.assertTransferCount(InvokeGRPC.REL_FAILURE, 0); + runner.assertPenalizeCount(1); + + final List noRetryFiles = runner.getFlowFilesForRelationship(InvokeGRPC.REL_NO_RETRY); + assertThat(noRetryFiles.size(), equalTo(1)); + final MockFlowFile noRetry = noRetryFiles.get(0); + noRetry.assertAttributeEquals(InvokeGRPC.RESPONSE_CODE, String.valueOf(FlowFileReply.ResponseCode.ERROR)); + noRetry.assertAttributeEquals(InvokeGRPC.RESPONSE_BODY, "error"); + noRetry.assertAttributeEquals(InvokeGRPC.SERVICE_HOST, TestGRPCServer.HOST); + noRetry.assertAttributeEquals(InvokeGRPC.SERVICE_PORT, String.valueOf(port)); + + final List responseFiles = runner.getFlowFilesForRelationship(InvokeGRPC.REL_RESPONSE); + assertThat(responseFiles.size(), equalTo(1)); + final MockFlowFile response = responseFiles.get(0); + response.assertAttributeEquals(InvokeGRPC.RESPONSE_CODE, String.valueOf(FlowFileReply.ResponseCode.ERROR)); + response.assertAttributeEquals(InvokeGRPC.RESPONSE_BODY, "error"); + response.assertAttributeEquals(InvokeGRPC.SERVICE_HOST, TestGRPCServer.HOST); + response.assertAttributeEquals(InvokeGRPC.SERVICE_PORT, String.valueOf(port)); + } finally { + server.stop(); + } + } + + @Test + public void testNoInput() throws Exception { + final TestGRPCServer server = new TestGRPCServer<>(DummyFlowFileService.class); + + try { + final int port = server.start(0); + + final TestRunner runner = TestRunners.newTestRunner(InvokeGRPC.class); + runner.setProperty(InvokeGRPC.PROP_SERVICE_HOST, TestGRPCServer.HOST); + runner.setProperty(InvokeGRPC.PROP_SERVICE_PORT, String.valueOf(port)); + + runner.run(); + runner.assertTransferCount(InvokeGRPC.REL_RESPONSE, 0); + runner.assertTransferCount(InvokeGRPC.REL_SUCCESS_REQ, 0); + runner.assertTransferCount(InvokeGRPC.REL_RETRY, 0); + runner.assertTransferCount(InvokeGRPC.REL_NO_RETRY, 0); + runner.assertTransferCount(InvokeGRPC.REL_FAILURE, 0); + runner.assertPenalizeCount(0); + } finally { + server.stop(); + } + } + + @Test + public void testServerConnectionFail() throws Exception { + + final int port = TestGRPCServer.randomPort(); + + // should be no gRPC server running @ that port, so processor will fail + final TestRunner runner = TestRunners.newTestRunner(InvokeGRPC.class); + runner.setProperty(InvokeGRPC.PROP_SERVICE_HOST, TestGRPCServer.HOST); + runner.setProperty(InvokeGRPC.PROP_SERVICE_PORT, Integer.toString(port)); + + final MockFlowFile mockFlowFile = new MockFlowFile(SUCCESS); + runner.enqueue(mockFlowFile); + runner.run(); + + runner.assertTransferCount(InvokeGRPC.REL_RESPONSE, 0); + runner.assertTransferCount(InvokeGRPC.REL_SUCCESS_REQ, 0); + runner.assertTransferCount(InvokeGRPC.REL_RETRY, 0); + runner.assertTransferCount(InvokeGRPC.REL_NO_RETRY, 0); + runner.assertTransferCount(InvokeGRPC.REL_FAILURE, 1); + + final List responseFiles = runner.getFlowFilesForRelationship(InvokeGRPC.REL_FAILURE); + assertThat(responseFiles.size(), equalTo(1)); + final MockFlowFile response = responseFiles.get(0); + response.assertAttributeEquals(InvokeGRPC.SERVICE_HOST, TestGRPCServer.HOST); + response.assertAttributeEquals(InvokeGRPC.SERVICE_PORT, Integer.toString(port)); + response.assertAttributeEquals(InvokeGRPC.EXCEPTION_CLASS, "io.grpc.StatusRuntimeException"); + + } + + /** + * Dummy gRPC service whose responses are dictated by the IDs on the messages it receives + */ + private static class DummyFlowFileService extends FlowFileServiceGrpc.FlowFileServiceImplBase { + + public DummyFlowFileService() { + } + + @Override + public void send(FlowFileRequest request, StreamObserver responseObserver) { + final FlowFileReply.Builder replyBuilder = FlowFileReply.newBuilder(); + // use the id to dictate response codes + final long id = request.getId(); + + if (id == ERROR) { + replyBuilder.setResponseCode(FlowFileReply.ResponseCode.ERROR) + .setBody("error"); + } else if (id == SUCCESS) { + replyBuilder.setResponseCode(FlowFileReply.ResponseCode.SUCCESS) + .setBody("success"); + } else if (id == RETRY) { + replyBuilder.setResponseCode(FlowFileReply.ResponseCode.RETRY) + .setBody("retry"); + // else, assume the request is to include the flowfile content in the response + } else { + replyBuilder.setResponseCode(FlowFileReply.ResponseCode.SUCCESS) + .setBody(request.getContent().toStringUtf8()); + + } + responseObserver.onNext(replyBuilder.build()); + responseObserver.onCompleted(); + } + } +} diff --git a/nifi-nar-bundles/nifi-grpc-bundle/nifi-grpc-processors/src/test/java/org/apache/nifi/processors/grpc/TestListenGRPC.java b/nifi-nar-bundles/nifi-grpc-bundle/nifi-grpc-processors/src/test/java/org/apache/nifi/processors/grpc/TestListenGRPC.java new file mode 100644 index 0000000000..7293c79e9b --- /dev/null +++ b/nifi-nar-bundles/nifi-grpc-bundle/nifi-grpc-processors/src/test/java/org/apache/nifi/processors/grpc/TestListenGRPC.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.grpc; + +import org.apache.nifi.reporting.InitializationException; +import org.apache.nifi.security.util.SslContextFactory; +import org.apache.nifi.security.util.TemporaryKeyStoreBuilder; +import org.apache.nifi.security.util.TlsConfiguration; +import org.apache.nifi.security.util.TlsException; +import org.apache.nifi.ssl.RestrictedSSLContextService; +import org.apache.nifi.util.TestRunner; +import org.apache.nifi.util.TestRunners; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import javax.net.SocketFactory; +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLSession; +import javax.net.ssl.SSLSocket; +import javax.net.ssl.SSLSocketFactory; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.Socket; + +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.when; + +@Timeout(10) +@ExtendWith(MockitoExtension.class) +class TestListenGRPC { + static final String LOCALHOST = "localhost"; + + static final String SSL_SERVICE_ID = RestrictedSSLContextService.class.getSimpleName(); + + static TlsConfiguration tlsConfiguration; + + static SSLContext sslContext; + + @Mock + RestrictedSSLContextService sslContextService; + + TestRunner runner; + + @BeforeAll + static void setTlsConfiguration() throws TlsException { + tlsConfiguration = new TemporaryKeyStoreBuilder().build(); + sslContext = SslContextFactory.createSslContext(tlsConfiguration); + } + + @BeforeEach + void setRunner() { + runner = TestRunners.newTestRunner(ListenGRPC.class); + } + + @Test + void testRunSocketListening() throws IOException { + runner.setProperty(ListenGRPC.PROP_SERVICE_PORT, "0"); + runner.assertValid(); + + runner.run(1, false); + final int port = ((ListenGRPC) runner.getProcessor()).getListeningPort(); + + assertSocketConnected(port, SocketFactory.getDefault()); + } + + @Test + void testRunSocketListeningSslContextService() throws IOException, InitializationException { + runner.setProperty(ListenGRPC.PROP_SERVICE_PORT, "0"); + + when(sslContextService.getIdentifier()).thenReturn(SSL_SERVICE_ID); + when(sslContextService.createTlsConfiguration()).thenReturn(tlsConfiguration); + when(sslContextService.createContext()).thenReturn(sslContext); + + runner.addControllerService(SSL_SERVICE_ID, sslContextService); + runner.enableControllerService(sslContextService); + + runner.setProperty(ListenGRPC.PROP_SSL_CONTEXT_SERVICE, SSL_SERVICE_ID); + runner.setProperty(ListenGRPC.PROP_USE_SECURE, Boolean.TRUE.toString()); + + runner.assertValid(); + + runner.run(1, false); + + final int port = ((ListenGRPC) runner.getProcessor()).getListeningPort(); + assertSocketConnectedProtocolNegotiated(port, sslContext.getSocketFactory()); + } + + private void assertSocketConnected(final int port, final SocketFactory socketFactory) throws IOException { + try (final Socket socket = socketFactory.createSocket()) { + assertSocketConnected(port, socket); + } + } + + private void assertSocketConnectedProtocolNegotiated(final int port, final SSLSocketFactory socketFactory) throws IOException { + try (final SSLSocket socket = (SSLSocket) socketFactory.createSocket()) { + assertSocketConnected(port, socket); + + socket.startHandshake(); + + final SSLSession session = socket.getSession(); + assertNotNull(session); + assertNotNull(session.getCipherSuite()); + } + } + + private void assertSocketConnected(final int port, final Socket socket) throws IOException { + final InetSocketAddress socketAddress = new InetSocketAddress(LOCALHOST, port); + socket.connect(socketAddress); + assertTrue(socket.isConnected()); + } +} diff --git a/nifi-nar-bundles/nifi-grpc-bundle/pom.xml b/nifi-nar-bundles/nifi-grpc-bundle/pom.xml new file mode 100644 index 0000000000..e11e97a8bd --- /dev/null +++ b/nifi-nar-bundles/nifi-grpc-bundle/pom.xml @@ -0,0 +1,49 @@ + + + + 4.0.0 + + + org.apache.nifi + nifi-nar-bundles + 1.28.0-SNAPSHOT + + + nifi-grpc-bundle + pom + A bundle of processors that speak the gRPC protocol + + + nifi-grpc-common + nifi-grpc-processors + nifi-grpc-nar + + + + 1.58.0 + 3.22.5 + + + + + + org.apache.nifi + nifi-grpc-processors + 1.28.0-SNAPSHOT + + + + diff --git a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/processgroupstatus/ProcessGroupStatusEnumerator.java b/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/processgroupstatus/ProcessGroupStatusEnumerator.java index b4cf4f6ccc..45e1016ceb 100644 --- a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/processgroupstatus/ProcessGroupStatusEnumerator.java +++ b/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/processgroupstatus/ProcessGroupStatusEnumerator.java @@ -112,12 +112,7 @@ public class ProcessGroupStatusEnumerator implements Enumerator { status.getTerminatedThreadCount(), status.getQueuedCount(), status.getVersionedFlowState() == null ? null : status.getVersionedFlowState().name(), - status.getProcessingNanos(), - status.getProcessingPerformanceStatus() == null ? -1 : status.getProcessingPerformanceStatus().getCpuDuration(), - status.getProcessingPerformanceStatus() == null ? -1 : status.getProcessingPerformanceStatus().getContentReadDuration(), - status.getProcessingPerformanceStatus() == null ? -1 : status.getProcessingPerformanceStatus().getContentWriteDuration(), - status.getProcessingPerformanceStatus() == null ? -1 : status.getProcessingPerformanceStatus().getSessionCommitDuration(), - status.getProcessingPerformanceStatus() == null ? -1 : status.getProcessingPerformanceStatus().getGarbageCollectionDuration() + status.getProcessingNanos() }; // If we want no fields just return null diff --git a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/processgroupstatus/ProcessGroupStatusTable.java b/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/processgroupstatus/ProcessGroupStatusTable.java index faf75c0e57..0790eba069 100644 --- a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/processgroupstatus/ProcessGroupStatusTable.java +++ b/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/processgroupstatus/ProcessGroupStatusTable.java @@ -172,12 +172,7 @@ public class ProcessGroupStatusTable extends AbstractTable implements QueryableT "terminatedThreadCount", "queuedCount", "versionedFlowState", - "processingNanos", - "cpuDuration", - "contentReadDuration", - "contentWriteDuration", - "sessionCommitDuration", - "garbageCollectionDuration" + "processingNanos" ); final List types = Arrays.asList( typeFactory.createJavaType(String.class), @@ -200,11 +195,6 @@ public class ProcessGroupStatusTable extends AbstractTable implements QueryableT typeFactory.createJavaType(int.class), typeFactory.createJavaType(int.class), typeFactory.createJavaType(String.class), - typeFactory.createJavaType(long.class), - typeFactory.createJavaType(long.class), - typeFactory.createJavaType(long.class), - typeFactory.createJavaType(long.class), - typeFactory.createJavaType(long.class), typeFactory.createJavaType(long.class) ); diff --git a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/processorstatus/ProcessorStatusEnumerator.java b/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/processorstatus/ProcessorStatusEnumerator.java index c16187ba04..4e550cef38 100644 --- a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/processorstatus/ProcessorStatusEnumerator.java +++ b/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/processorstatus/ProcessorStatusEnumerator.java @@ -114,12 +114,7 @@ public class ProcessorStatusEnumerator implements Enumerator { status.getInvocations(), status.getProcessingNanos(), status.getRunStatus().name(), - status.getExecutionNode() == null ? null : status.getExecutionNode().name(), - status.getProcessingPerformanceStatus() == null ? -1 : status.getProcessingPerformanceStatus().getCpuDuration(), - status.getProcessingPerformanceStatus() == null ? -1 : status.getProcessingPerformanceStatus().getContentReadDuration(), - status.getProcessingPerformanceStatus() == null ? -1 : status.getProcessingPerformanceStatus().getContentWriteDuration(), - status.getProcessingPerformanceStatus() == null ? -1 : status.getProcessingPerformanceStatus().getSessionCommitDuration(), - status.getProcessingPerformanceStatus() == null ? -1 : status.getProcessingPerformanceStatus().getGarbageCollectionDuration() + status.getExecutionNode() == null ? null : status.getExecutionNode().name() }; // If we want no fields just return null diff --git a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/processorstatus/ProcessorStatusTable.java b/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/processorstatus/ProcessorStatusTable.java index ebc995f67b..13b2207c61 100644 --- a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/processorstatus/ProcessorStatusTable.java +++ b/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/processorstatus/ProcessorStatusTable.java @@ -173,12 +173,7 @@ public class ProcessorStatusTable extends AbstractTable implements QueryableTabl "invocations", "processingNanos", "runStatus", - "executionNode", - "cpuDuration", - "contentReadDuration", - "contentWriteDuration", - "sessionCommitDuration", - "garbageCollectionDuration" + "executionNode" ); final List types = Arrays.asList( typeFactory.createJavaType(String.class), @@ -202,12 +197,7 @@ public class ProcessorStatusTable extends AbstractTable implements QueryableTabl typeFactory.createJavaType(int.class), typeFactory.createJavaType(long.class), typeFactory.createJavaType(String.class), - typeFactory.createJavaType(String.class), - typeFactory.createJavaType(long.class), - typeFactory.createJavaType(long.class), - typeFactory.createJavaType(long.class), - typeFactory.createJavaType(long.class), - typeFactory.createJavaType(long.class) + typeFactory.createJavaType(String.class) ); relDataType = typeFactory.createStructType(Pair.zip(names, types)); diff --git a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/test/java/org/apache/nifi/reporting/sql/TestQueryNiFiReportingTask.java b/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/test/java/org/apache/nifi/reporting/sql/TestQueryNiFiReportingTask.java index fb8bf59ebd..7eccb29337 100644 --- a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/test/java/org/apache/nifi/reporting/sql/TestQueryNiFiReportingTask.java +++ b/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/test/java/org/apache/nifi/reporting/sql/TestQueryNiFiReportingTask.java @@ -509,7 +509,7 @@ class TestQueryNiFiReportingTask { assertEquals(4, rows.size()); // Validate the first row Map row = rows.get(0); - assertEquals(26, row.size()); + assertEquals(21, row.size()); assertEquals(1L, row.get("bytesRead")); // Validate the second row row = rows.get(1);