NIFI-10312 Fixed MiNiFi C2 status and Flow ID communication

This closes #6281

Signed-off-by: David Handermann <exceptionfactory@apache.org>

c2/c2-client-bundle/c2-client-base/src/main/java/org/apache/nifi/c2/serializer/C2JacksonSerializer.java

@@ -21,6 +21,9 @@ import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.DeserializationFeature;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import java.util.Optional;
+
+import com.fasterxml.jackson.databind.module.SimpleModule;
+import org.apache.nifi.c2.protocol.api.OperandType;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -35,6 +38,10 @@ public class C2JacksonSerializer implements C2Serializer {
         objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
         objectMapper.setSerializationInclusion(JsonInclude.Include.NON_NULL);
         objectMapper.enable(DeserializationFeature.READ_ENUMS_USING_TO_STRING);
+
+        SimpleModule module = new SimpleModule();
+        module.addDeserializer(OperandType.class, new OperandTypeDeserializer());
+        objectMapper.registerModule(module);
     }
 
     @Override

c2/c2-client-bundle/c2-client-base/src/main/java/org/apache/nifi/c2/serializer/OperandTypeDeserializer.java

@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.c2.serializer;
+
+import com.fasterxml.jackson.core.JacksonException;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.deser.std.StdDeserializer;
+import org.apache.nifi.c2.protocol.api.OperandType;
+
+import java.io.IOException;
+
+public class OperandTypeDeserializer extends StdDeserializer<OperandType> {
+
+    public OperandTypeDeserializer() {
+        this(null);
+    }
+
+    public OperandTypeDeserializer(Class<?> vc) {
+        super(vc);
+    }
+
+    @Override
+    public OperandType deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) throws IOException, JacksonException {
+        JsonNode node = jsonParser.getCodec().readTree(jsonParser);
+
+        return OperandType.fromString(node.textValue()).orElse(null);
+    }
+}

c2/c2-client-bundle/c2-client-service/src/main/java/org/apache/nifi/c2/client/service/operation/UpdateConfigurationOperationHandler.java

@@ -21,6 +21,7 @@ import static org.apache.nifi.c2.protocol.api.OperandType.CONFIGURATION;
 import static org.apache.nifi.c2.protocol.api.OperationType.UPDATE;
 
 import java.net.URI;
+import java.util.Map;
 import java.util.Optional;
 import java.util.function.Function;
 import java.util.regex.Matcher;
@@ -36,10 +37,9 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 public class UpdateConfigurationOperationHandler implements C2OperationHandler {
-
     private static final Logger logger = LoggerFactory.getLogger(UpdateConfigurationOperationHandler.class);
     private static final Pattern FLOW_ID_PATTERN = Pattern.compile("/[^/]+?/[^/]+?/[^/]+?/([^/]+)?/?.*");
-
+    static final String FLOW_ID = "flowId";
     static final String LOCATION = "location";
 
     private final C2Client client;
@@ -75,30 +75,44 @@ public class UpdateConfigurationOperationHandler implements C2OperationHandler {
             .map(map -> map.get(LOCATION))
             .orElse(EMPTY);
 
-        String newFlowId = parseFlowId(updateLocation);
-        if (flowIdHolder.getFlowId() == null || !flowIdHolder.getFlowId().equals(newFlowId)) {
-            logger.info("Will perform flow update from {} for operation #{}. Previous flow id was {}, replacing with new id {}", updateLocation, opIdentifier,
-                flowIdHolder.getFlowId() == null ? "not set" : flowIdHolder.getFlowId(), newFlowId);
+        String flowId = getFlowId(operation.getArgs(), updateLocation);
+        if (flowId == null) {
+            state.setState(C2OperationState.OperationState.NOT_APPLIED);
+            state.setDetails("Could not get flowId from the operation.");
+            logger.info("FlowId is missing, no update will be performed.");
         } else {
-            logger.info("Flow is current, no update is necessary...");
+            if (flowIdHolder.getFlowId() == null || !flowIdHolder.getFlowId().equals(flowId)) {
+                logger.info("Will perform flow update from {} for operation #{}. Previous flow id was {}, replacing with new id {}", updateLocation, opIdentifier,
+                        flowIdHolder.getFlowId() == null ? "not set" : flowIdHolder.getFlowId(), flowId);
+            } else {
+                logger.info("Flow is current, no update is necessary...");
+            }
+            flowIdHolder.setFlowId(flowId);
+            state.setState(updateFlow(opIdentifier, updateLocation));
         }
+        return operationAck;
+    }
 
-        flowIdHolder.setFlowId(newFlowId);
+    private C2OperationState.OperationState updateFlow(String opIdentifier, String updateLocation) {
         Optional<byte[]> updateContent = client.retrieveUpdateContent(updateLocation);
         if (updateContent.isPresent()) {
             if (updateFlow.apply(updateContent.get())) {
-                state.setState(C2OperationState.OperationState.FULLY_APPLIED);
                 logger.debug("Update configuration applied for operation #{}.", opIdentifier);
+                return C2OperationState.OperationState.FULLY_APPLIED;
             } else {
-                state.setState(C2OperationState.OperationState.NOT_APPLIED);
                 logger.error("Update resulted in error for operation #{}.", opIdentifier);
+                return C2OperationState.OperationState.NOT_APPLIED;
             }
         } else {
-            state.setState(C2OperationState.OperationState.NOT_APPLIED);
             logger.error("Update content retrieval resulted in empty content so flow update was omitted for operation #{}.", opIdentifier);
+            return C2OperationState.OperationState.NOT_APPLIED;
         }
+    }
 
-        return operationAck;
+    private String getFlowId(Map<String, String> args, String updateLocation) {
+        return Optional.ofNullable(args)
+        .map(map -> map.get(FLOW_ID))
+        .orElseGet(() -> parseFlowId(updateLocation));
     }
 
     private String parseFlowId(String flowUpdateUrl) {
@@ -108,11 +122,10 @@ public class UpdateConfigurationOperationHandler implements C2OperationHandler {
 
             if (matcher.matches()) {
                 return matcher.group(1);
-            } else {
-                throw new IllegalArgumentException(String.format("Flow Update URL format unexpected [%s]", flowUpdateUrl));
             }
         } catch (Exception e) {
-            throw new IllegalStateException("Could not get flow id from the provided URL", e);
+            logger.error("Could not get flow id from the provided URL, flow update URL format unexpected [{}]", flowUpdateUrl);
         }
+        return null;
     }
 }

c2/c2-client-bundle/c2-client-service/src/test/java/org/apache/nifi/c2/client/service/operation/UpdateConfigurationOperationHandlerTest.java

@@ -18,12 +18,13 @@ package org.apache.nifi.c2.client.service.operation;
 
 import static org.apache.commons.lang3.StringUtils.EMPTY;
 import static org.apache.nifi.c2.client.service.operation.UpdateConfigurationOperationHandler.LOCATION;
+import static org.apache.nifi.c2.client.service.operation.UpdateConfigurationOperationHandler.FLOW_ID;
 import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.Mockito.when;
 
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.Map;
 import java.util.Optional;
 import java.util.function.Function;
@@ -41,8 +42,6 @@ import org.mockito.junit.jupiter.MockitoExtension;
 
 @ExtendWith(MockitoExtension.class)
 public class UpdateConfigurationOperationHandlerTest {
-
-    private static final String FLOW_ID = "flowId";
     private static final String OPERATION_ID = "operationId";
     private static final Map<String, String> CORRECT_LOCATION_MAP = Collections.singletonMap(LOCATION, "/path/for/the/" + FLOW_ID);
     private static final Map<String, String> INCORRECT_LOCATION_MAP = Collections.singletonMap(LOCATION, "incorrect/location");
@@ -62,12 +61,34 @@ public class UpdateConfigurationOperationHandlerTest {
     }
 
     @Test
-    void testHandleThrowsExceptionForIncorrectArg() {
+    void testHandleIncorrectArg() {
         UpdateConfigurationOperationHandler handler = new UpdateConfigurationOperationHandler(null, null, null);
         C2Operation operation = new C2Operation();
         operation.setArgs(INCORRECT_LOCATION_MAP);
 
-        IllegalStateException exception = assertThrows(IllegalStateException.class, () -> handler.handle(operation));
+        C2OperationAck response = handler.handle(operation);
+
+        assertEquals(C2OperationState.OperationState.NOT_APPLIED, response.getOperationState().getState());
+    }
+
+    @Test
+    void testHandleFlowIdInArg() {
+        Function<byte[], Boolean> successUpdate = x -> true;
+        when(flowIdHolder.getFlowId()).thenReturn(FLOW_ID);
+        when(client.retrieveUpdateContent(any())).thenReturn(Optional.of("content".getBytes()));
+        UpdateConfigurationOperationHandler handler = new UpdateConfigurationOperationHandler(client, flowIdHolder, successUpdate);
+        C2Operation operation = new C2Operation();
+        operation.setIdentifier(OPERATION_ID);
+
+        Map<String, String> args = new HashMap<>();
+        args.putAll(INCORRECT_LOCATION_MAP);
+        args.put(FLOW_ID, "argsFlowId");
+        operation.setArgs(args);
+
+        C2OperationAck response = handler.handle(operation);
+
+        assertEquals(OPERATION_ID, response.getOperationId());
+        assertEquals(C2OperationState.OperationState.FULLY_APPLIED, response.getOperationState().getState());
     }
 
     @Test

c2/c2-protocol/c2-protocol-api/src/main/java/org/apache/nifi/c2/protocol/api/OperandType.java

@@ -39,4 +39,3 @@ public enum OperandType {
         return super.toString().toLowerCase();
     }
 }
-

minifi/minifi-c2/minifi-c2-service/src/main/java/org/apache/nifi/minifi/c2/service/ConfigService.java

@@ -228,8 +228,8 @@ public class ConfigService {
                 try {
                     configuration = configurationProviderValue.getConfiguration();
                 } catch (ConfigurationProviderException cpe) {
-                    logger.warn("No flow available for agent class " + agentClass + ", returning No Content (204)");
-                    response = Response.noContent().build();
+                    logger.warn("No flow available for agent class " + agentClass + ", returning OK (200) with no update request");
+                    response = Response.ok(new C2HeartbeatResponse()).build();
                     return response;
                 }
                 try (InputStream inputStream = configuration.getInputStream();

minifi/minifi-c2/minifi-c2-service/src/main/java/org/apache/nifi/minifi/c2/service/SimpleC2ProtocolService.java

@@ -42,6 +42,8 @@ public class SimpleC2ProtocolService implements C2ProtocolService {
     private static final Logger logger = LoggerFactory.getLogger(SimpleC2ProtocolService.class);
 
     private static final Set<String> issuedOperationIds = new HashSet<>();
+    private static final String LOCATION = "location";
+    private static final String FLOW_ID = "flowId";
 
     private final Map<String, String> currentFlowIds;
 
@@ -108,7 +110,10 @@ public class SimpleC2ProtocolService implements C2ProtocolService {
             c2Operation.setIdentifier(operationID);
             c2Operation.setOperation(OperationType.UPDATE);
             c2Operation.setOperand(OperandType.CONFIGURATION);
-            c2Operation.setArgs(Collections.singletonMap("location", context.getBaseUri().toString()));
+            Map<String, String> args = new HashMap<>();
+            args.put(LOCATION, context.getBaseUri().toString());
+            args.put(FLOW_ID, context.getSha256());
+            c2Operation.setArgs(args);
             List<C2Operation> requestedOperations = Collections.singletonList(c2Operation);
             c2HeartbeatResponse.setRequestedOperations(requestedOperations);
             currentFlowIds.put(heartbeat.getAgentId(), context.getSha256());

minifi/minifi-integration-tests/src/test/java/org/apache/nifi/minifi/integration/c2/C2ProtocolIntegrationTest.java

@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.minifi.integration.c2;
+
+import com.palantir.docker.compose.DockerComposeExtension;
+import com.palantir.docker.compose.connection.waiting.HealthChecks;
+import org.apache.nifi.minifi.integration.util.LogUtil;
+import org.apache.nifi.security.util.KeystoreType;
+import org.apache.nifi.security.util.SslContextFactory;
+import org.apache.nifi.security.util.StandardTlsConfiguration;
+import org.apache.nifi.security.util.TlsConfiguration;
+import org.apache.nifi.toolkit.tls.standalone.TlsToolkitStandalone;
+import org.apache.nifi.toolkit.tls.standalone.TlsToolkitStandaloneCommandLine;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.Timeout;
+
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.SSLSocketFactory;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+@Timeout(60)
+public class C2ProtocolIntegrationTest {
+    private static final String AGENT_1 = "minifi-edge1";
+    private static final String AGENT_2 = "minifi-edge2";
+    private static final String AGENT_3 = "minifi-edge3";
+    private static final String AGENT_CLASS_1 = "raspi3";
+    private static final String AGENT_CLASS_2 = "raspi4";
+    private static final String SERVICE = "c2-authoritative";
+    private static final String CONFIG_YAML = "config.text.yml.v2";
+    private static Path certificatesDirectory;
+    private static SSLContext trustSslContext;
+    private static SSLSocketFactory healthCheckSocketFactory;
+    public static DockerComposeExtension docker = DockerComposeExtension.builder()
+            .file("target/test-classes/docker-compose-c2-protocol.yml")
+            .waitingForService(AGENT_1, HealthChecks.toRespond2xxOverHttp(8000, dockerPort -> "http://" + dockerPort.getIp() + ":" + dockerPort.getExternalPort()))
+            .waitingForService(AGENT_2, HealthChecks.toRespond2xxOverHttp(8000, dockerPort -> "http://" + dockerPort.getIp() + ":" + dockerPort.getExternalPort()))
+            .waitingForService(AGENT_3, HealthChecks.toRespond2xxOverHttp(8000, dockerPort -> "http://" + dockerPort.getIp() + ":" + dockerPort.getExternalPort()))
+            .build();
+
+    private static Path resourceDirectory;
+    private static Path authoritativeFiles;
+    private static Path minifiEdge1Version2;
+    private static Path minifiEdge2Version2;
+    private static Path minifiEdge3Version2;
+
+    /**
+     * Generates certificates with the tls-toolkit and then starts up the docker compose file
+     */
+    @BeforeAll
+    public static void init() throws Exception {
+        resourceDirectory = Paths.get(C2ProtocolIntegrationTest.class.getClassLoader()
+                .getResource("docker-compose-c2-protocol.yml").getFile()).getParent();
+        certificatesDirectory = resourceDirectory.toAbsolutePath().resolve("certificates-c2-protocol");
+        authoritativeFiles = resourceDirectory.resolve("c2").resolve("protocol").resolve(SERVICE).resolve("files");
+        minifiEdge1Version2 = authoritativeFiles.resolve("edge1").resolve(AGENT_CLASS_1).resolve(CONFIG_YAML);
+        minifiEdge2Version2 = authoritativeFiles.resolve("edge2").resolve(AGENT_CLASS_1).resolve(CONFIG_YAML);
+        minifiEdge3Version2 = authoritativeFiles.resolve("edge3").resolve(AGENT_CLASS_2).resolve(CONFIG_YAML);
+
+        if (Files.exists(minifiEdge1Version2)) {
+            Files.delete(minifiEdge1Version2);
+        }
+        if (Files.exists(minifiEdge2Version2)) {
+            Files.delete(minifiEdge2Version2);
+        }
+        if (Files.exists(minifiEdge3Version2)) {
+            Files.delete(minifiEdge3Version2);
+        }
+
+        List<String> toolkitCommandLine = new ArrayList<>(Arrays.asList("-O", "-o", certificatesDirectory.toFile().getAbsolutePath(), "-S", "badKeystorePass", "-P", "badTrustPass"));
+        for (String serverHostname : Arrays.asList(SERVICE, AGENT_1, AGENT_2, AGENT_3)) {
+            toolkitCommandLine.add("-n");
+            toolkitCommandLine.add(serverHostname);
+        }
+        Files.createDirectories(certificatesDirectory);
+        TlsToolkitStandaloneCommandLine tlsToolkitStandaloneCommandLine = new TlsToolkitStandaloneCommandLine();
+        tlsToolkitStandaloneCommandLine.parse(toolkitCommandLine.toArray(new String[toolkitCommandLine.size()]));
+        new TlsToolkitStandalone().createNifiKeystoresAndTrustStores(tlsToolkitStandaloneCommandLine.createConfig());
+
+        TlsConfiguration tlsConfiguration = new StandardTlsConfiguration(
+                null,null,null,
+                certificatesDirectory.resolve(SERVICE).resolve("truststore.jks").toFile().getAbsolutePath(),
+                "badTrustPass",
+                KeystoreType.JKS);
+        trustSslContext = SslContextFactory.createSslContext(tlsConfiguration);
+        healthCheckSocketFactory = trustSslContext.getSocketFactory();
+
+        docker.before();
+    }
+
+    @AfterAll
+    public static void stopDocker() {
+        docker.after();
+    }
+
+    @Test
+    public void testFlowPublishThroughC2Protocol() throws Exception {
+        LogUtil.verifyLogEntries("c2/protocol/minifi-edge1/expected.json", docker.containers().container(AGENT_1));
+        LogUtil.verifyLogEntries("c2/protocol/minifi-edge2/expected.json", docker.containers().container(AGENT_2));
+        LogUtil.verifyLogEntries("c2/protocol/minifi-edge3/expected.json", docker.containers().container(AGENT_3));
+    }
+}

minifi/minifi-integration-tests/src/test/resources/c2/protocol/c2-authoritative/conf/authorities.yaml

@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the \"License\"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an \"AS IS\" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+CN=minifi-edge1, OU=NIFI:
+  - EDGE_1
+CN=minifi-edge2, OU=NIFI:
+  - EDGE_2
+CN=minifi-edge3, OU=NIFI:
+  - EDGE_3

minifi/minifi-integration-tests/src/test/resources/c2/protocol/c2-authoritative/conf/authorizations.yaml

@@ -0,0 +1,52 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the \"License\"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an \"AS IS\" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+Default Action: deny
+Paths:
+  /c2/config:
+    Default Action: deny
+    Actions:
+    - Authorization: EDGE_1
+      Query Parameters:
+        class: raspi3
+      Action: allow
+    - Authorization: EDGE_2
+      Query Parameters:
+        class: raspi3
+      Action: allow
+    - Authorization: EDGE_3
+      Query Parameters:
+        class: raspi4
+      Action: allow
+
+  /c2/config/heartbeat:
+    Default Action: deny
+    Actions:
+      - Authorization: EDGE_1
+        Action: allow
+      - Authorization: EDGE_2
+        Action: allow
+      - Authorization: EDGE_3
+        Action: allow
+
+  /c2/config/acknowledge:
+    Default Action: deny
+    Actions:
+      - Authorization: EDGE_1
+        Action: allow
+      - Authorization: EDGE_2
+        Action: allow
+      - Authorization: EDGE_3
+        Action: allow

minifi/minifi-integration-tests/src/test/resources/c2/protocol/c2-authoritative/conf/c2.properties

@@ -0,0 +1,27 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+minifi.c2.server.port=10443
+
+minifi.c2.server.secure=true
+minifi.c2.server.keystore=./conf/keystore.jks
+minifi.c2.server.keystoreType=JKS
+minifi.c2.server.keystorePasswd=badKeystorePass
+minifi.c2.server.keyPasswd=badKeystorePass
+minifi.c2.server.truststore=./conf/truststore.jks
+minifi.c2.server.truststoreType=JKS
+minifi.c2.server.truststorePasswd=badTrustPass

minifi/minifi-integration-tests/src/test/resources/c2/protocol/c2-authoritative/conf/minifi-c2-context.xml

@@ -0,0 +1,63 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+<beans default-lazy-init="true"
+       xmlns="http://www.springframework.org/schema/beans"
+       xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+       xmlns:util="http://www.springframework.org/schema/util"
+       xmlns:context="http://www.springframework.org/schema/context"
+       xmlns:aop="http://www.springframework.org/schema/aop"
+       xsi:schemaLocation="http://www.springframework.org/schema/beans https://www.springframework.org/schema/beans/spring-beans-3.1.xsd
+    http://www.springframework.org/schema/util https://www.springframework.org/schema/util/spring-util-3.1.xsd
+    http://www.springframework.org/schema/context https://www.springframework.org/schema/context/spring-context-3.1.xsd
+    http://www.springframework.org/schema/aop https://www.springframework.org/schema/aop/spring-aop-3.1.xsd">
+
+    <bean id="configService" class="org.apache.nifi.minifi.c2.service.ConfigService" scope="singleton">
+        <constructor-arg>
+            <list>
+                <bean class="org.apache.nifi.minifi.c2.provider.cache.CacheConfigurationProvider">
+                    <constructor-arg>
+                        <list>
+                            <value>text/yml</value>
+                        </list>
+                    </constructor-arg>
+                    <constructor-arg>
+                        <bean class="org.apache.nifi.minifi.c2.cache.filesystem.FileSystemConfigurationCache">
+                            <constructor-arg>
+                                <value>./files</value>
+                            </constructor-arg>
+                            <constructor-arg>
+                                <value>${class}/config</value>
+                            </constructor-arg>
+                        </bean>
+                    </constructor-arg>
+                </bean>
+            </list>
+        </constructor-arg>
+        <constructor-arg>
+            <bean class="org.apache.nifi.minifi.c2.security.authorization.GrantedAuthorityAuthorizer">
+                <constructor-arg value="classpath:authorizations.yaml"/>
+            </bean>
+        </constructor-arg>
+        <constructor-arg>
+            <value>1000</value>
+        </constructor-arg>
+        <constructor-arg>
+            <value>1000</value>
+        </constructor-arg>
+    </bean>
+</beans>

minifi/minifi-integration-tests/src/test/resources/c2/protocol/c2-authoritative/files/raspi3/config.text.yml.v1

@@ -0,0 +1,129 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the \"License\"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an \"AS IS\" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+MiNiFi Config Version: 3
+Flow Controller:
+  name: Edge raspi3 v1.0
+  comment: ''
+Core Properties:
+  flow controller graceful shutdown period: 10 sec
+  flow service write delay interval: 500 ms
+  administrative yield duration: 30 sec
+  bored yield duration: 10 millis
+  max concurrent threads: 1
+  variable registry properties: ''
+FlowFile Repository:
+  implementation: org.apache.nifi.controller.repository.WriteAheadFlowFileRepository
+  partitions: 256
+  checkpoint interval: 2 mins
+  always sync: false
+  Swap:
+    threshold: 20000
+    in period: 5 sec
+    in threads: 1
+    out period: 5 sec
+    out threads: 4
+Content Repository:
+  implementation: org.apache.nifi.controller.repository.FileSystemRepository
+  content claim max appendable size: 10 MB
+  content claim max flow files: 100
+  content repository archive enabled: false
+  content repository archive max retention period: 12 hours
+  content repository archive max usage percentage: 50%
+  always sync: false
+Provenance Repository:
+  provenance rollover time: 1 min
+  implementation: org.apache.nifi.provenance.WriteAheadProvenanceRepository
+  provenance index shard size: 500 MB
+  provenance max storage size: 1 GB
+  provenance max storage time: 24 hours
+  provenance buffer size: 10000
+Component Status Repository:
+  buffer size: 1440
+  snapshot frequency: 1 min
+Security Properties:
+  keystore: ''
+  keystore type: ''
+  keystore password: ''
+  key password: ''
+  truststore: ''
+  truststore type: ''
+  truststore password: ''
+  ssl protocol: ''
+  Sensitive Props:
+    key: ''
+    algorithm: NIFI_PBKDF2_AES_GCM_256
+Processors:
+- id: c6a06b10-0af2-423b-9927-0bb8e6e04bc3
+  name: GenerateTestText
+  class: org.apache.nifi.processors.standard.GenerateFlowFile
+  max concurrent tasks: 1
+  scheduling strategy: TIMER_DRIVEN
+  scheduling period: 1000 ms
+  penalization period: 30000 ms
+  yield period: 1000 ms
+  run duration nanos: 0
+  auto-terminated relationships list: [
+    ]
+  Properties:
+    Batch Size: '1'
+    Data Format: Text
+    File Size: 0B
+    Unique FlowFiles: 'false'
+    character-set: UTF-8
+    generate-ff-custom-text: __testTextRaspi3__
+- id: 186322b1-4778-40e4-ba9d-a0a511d762f7
+  name: LogAttribute
+  class: org.apache.nifi.processors.standard.LogAttribute
+  max concurrent tasks: 1
+  scheduling strategy: TIMER_DRIVEN
+  scheduling period: 1000 ms
+  penalization period: 30000 ms
+  yield period: 1000 ms
+  run duration nanos: 0
+  auto-terminated relationships list:
+  - success
+  Properties:
+    Log FlowFile Properties: 'true'
+    Log Level: info
+    Log Payload: 'true'
+    Output Format: Line per Attribute
+    attributes-to-log-regex: .*
+    character-set: UTF-8
+Controller Services: [
+  ]
+Process Groups: [
+  ]
+Input Ports: [
+  ]
+Output Ports: [
+  ]
+Funnels: [
+  ]
+Connections:
+- id: 641b0183-8653-4988-9a74-bcf9780b1397
+  name: GenerateTestText/success/LogAttribute
+  source id: c6a06b10-0af2-423b-9927-0bb8e6e04bc3
+  source relationship names:
+  - success
+  destination id: 186322b1-4778-40e4-ba9d-a0a511d762f7
+  max work queue size: 10000
+  max work queue data size: 10 MB
+  flowfile expiration: 0 seconds
+  queue prioritizer class: ''
+Remote Process Groups: [
+  ]
+NiFi Properties Overrides: {
+  }

minifi/minifi-integration-tests/src/test/resources/c2/protocol/c2-authoritative/files/raspi4/config.text.yml.v1

@@ -0,0 +1,129 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the \"License\"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an \"AS IS\" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+MiNiFi Config Version: 3
+Flow Controller:
+  name: Edge raspi4 v1.0
+  comment: ''
+Core Properties:
+  flow controller graceful shutdown period: 10 sec
+  flow service write delay interval: 500 ms
+  administrative yield duration: 30 sec
+  bored yield duration: 10 millis
+  max concurrent threads: 1
+  variable registry properties: ''
+FlowFile Repository:
+  implementation: org.apache.nifi.controller.repository.WriteAheadFlowFileRepository
+  partitions: 256
+  checkpoint interval: 2 mins
+  always sync: false
+  Swap:
+    threshold: 20000
+    in period: 5 sec
+    in threads: 1
+    out period: 5 sec
+    out threads: 4
+Content Repository:
+  implementation: org.apache.nifi.controller.repository.FileSystemRepository
+  content claim max appendable size: 10 MB
+  content claim max flow files: 100
+  content repository archive enabled: false
+  content repository archive max retention period: 12 hours
+  content repository archive max usage percentage: 50%
+  always sync: false
+Provenance Repository:
+  provenance rollover time: 1 min
+  implementation: org.apache.nifi.provenance.WriteAheadProvenanceRepository
+  provenance index shard size: 500 MB
+  provenance max storage size: 1 GB
+  provenance max storage time: 24 hours
+  provenance buffer size: 10000
+Component Status Repository:
+  buffer size: 1440
+  snapshot frequency: 1 min
+Security Properties:
+  keystore: ''
+  keystore type: ''
+  keystore password: ''
+  key password: ''
+  truststore: ''
+  truststore type: ''
+  truststore password: ''
+  ssl protocol: ''
+  Sensitive Props:
+    key: ''
+    algorithm: NIFI_PBKDF2_AES_GCM_256
+Processors:
+- id: 6ef15904-e69e-425b-b4a9-427c367220a3
+  name: GenerateFlowFile
+  class: org.apache.nifi.processors.standard.GenerateFlowFile
+  max concurrent tasks: 1
+  scheduling strategy: TIMER_DRIVEN
+  scheduling period: 1000 ms
+  penalization period: 30000 ms
+  yield period: 1000 ms
+  run duration nanos: 0
+  auto-terminated relationships list: [
+    ]
+  Properties:
+    Batch Size: '1'
+    Data Format: Text
+    File Size: 0B
+    Unique FlowFiles: 'false'
+    character-set: UTF-8
+    generate-ff-custom-text: __testTextRaspi4__
+- id: 26f9038d-2cd9-4df3-a174-c48dda90fce7
+  name: LogAttribute
+  class: org.apache.nifi.processors.standard.LogAttribute
+  max concurrent tasks: 1
+  scheduling strategy: TIMER_DRIVEN
+  scheduling period: 0 ms
+  penalization period: 30000 ms
+  yield period: 1000 ms
+  run duration nanos: 0
+  auto-terminated relationships list:
+  - success
+  Properties:
+    Log FlowFile Properties: 'true'
+    Log Level: info
+    Log Payload: 'true'
+    Output Format: Line per Attribute
+    attributes-to-log-regex: .*
+    character-set: UTF-8
+Controller Services: [
+  ]
+Process Groups: [
+  ]
+Input Ports: [
+  ]
+Output Ports: [
+  ]
+Funnels: [
+  ]
+Connections:
+- id: 68ebc161-1b82-472b-a2f6-ee4173033f60
+  name: GenerateFlowFile/success/LogAttribute
+  source id: 6ef15904-e69e-425b-b4a9-427c367220a3
+  source relationship names:
+  - success
+  destination id: 26f9038d-2cd9-4df3-a174-c48dda90fce7
+  max work queue size: 10000
+  max work queue data size: 10 MB
+  flowfile expiration: 0 seconds
+  queue prioritizer class: ''
+Remote Process Groups: [
+  ]
+NiFi Properties Overrides: {
+  }

minifi/minifi-integration-tests/src/test/resources/c2/protocol/minifi-edge1/bootstrap.conf

@@ -0,0 +1,125 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# C2 Properties
+c2.enable=true
+c2.rest.url=https://c2-authoritative:10443/c2/config/heartbeat
+c2.rest.url.ack=https://c2-authoritative:10443/c2/config/acknowledge
+c2.rest.connectionTimeout=5 sec
+c2.rest.readTimeout=5 sec
+c2.rest.callTimeout=10 sec
+c2.agent.heartbeat.period=2000
+c2.agent.class=raspi3
+c2.config.directory=./conf
+c2.runtime.manifest.identifier=minifi
+c2.runtime.type=minifi-java
+c2.security.truststore.location=./conf/truststore.jks
+c2.security.truststore.password=badTrustPass
+c2.security.truststore.type=JKS
+c2.security.keystore.location=./conf/keystore.jks
+c2.security.keystore.password=badKeystorePass
+c2.security.keystore.type=JKS
+
+# Java command to use when running MiNiFi
+java=java
+
+# Username to use when running MiNiFi. This value will be ignored on Windows.
+run.as=
+
+# Configure where MiNiFi's lib and conf directories live
+lib.dir=./lib
+conf.dir=./conf
+
+# How long to wait after telling MiNiFi to shutdown before explicitly killing the Process
+graceful.shutdown.seconds=20
+
+# The location for the configuration file
+nifi.minifi.config=./conf/config.yml
+
+# Notifiers to use for the associated agent, comma separated list of class names
+nifi.minifi.notifier.ingestors=org.apache.nifi.minifi.bootstrap.configuration.ingestors.FileChangeIngestor
+#nifi.minifi.notifier.ingestors=org.apache.nifi.minifi.bootstrap.configuration.ingestors.RestChangeIngestor
+#nifi.minifi.notifier.ingestors=org.apache.nifi.minifi.bootstrap.configuration.ingestors.PullHttpChangeIngestor
+
+# File change notifier configuration
+
+# Path of the file to monitor for changes.  When these occur, the FileChangeNotifier, if configured, will begin the configuration reloading process
+nifi.minifi.notifier.ingestors.file.config.path=./conf/config-new.yml
+# How frequently the file specified by 'nifi.minifi.notifier.file.config.path' should be evaluated for changes.
+nifi.minifi.notifier.ingestors.file.polling.period.seconds=2
+
+# Rest change notifier configuration
+
+# Port on which the Jetty server will bind to, keep commented for a random open port
+#nifi.minifi.notifier.ingestors.receive.http.port=8338
+
+#Pull HTTP change notifier configuration
+
+# Hostname on which to pull configurations from
+# nifi.minifi.notifier.ingestors.pull.http.hostname=c2-authoritative
+# Port on which to pull configurations from
+# nifi.minifi.notifier.ingestors.pull.http.port=10443
+# Path to pull configurations from
+# nifi.minifi.notifier.ingestors.pull.http.path=/c2/config
+# Query string to pull configurations with
+# nifi.minifi.notifier.ingestors.pull.http.query=net=edge1&class=raspi3
+# Period on which to pull configurations from, defaults to 5 minutes if commented out
+# nifi.minifi.notifier.ingestors.pull.http.period.ms=3000
+
+# nifi.minifi.notifier.ingestors.pull.http.keystore.location=./conf/keystore.jks
+# nifi.minifi.notifier.ingestors.pull.http.keystore.type=JKS
+# nifi.minifi.notifier.ingestors.pull.http.keystore.password=badKeystorePass
+# nifi.minifi.notifier.ingestors.pull.http.truststore.location=./conf/truststore.jks
+# nifi.minifi.notifier.ingestors.pull.http.truststore.type=JKS
+# nifi.minifi.notifier.ingestors.pull.http.truststore.password=badTrustPass
+
+# Periodic Status Reporters to use for the associated agent, comma separated list of class names
+#nifi.minifi.status.reporter.components=org.apache.nifi.minifi.bootstrap.status.reporters.StatusLogger
+
+# Periodic Status Logger configuration
+
+# The FlowStatus query to submit to the MiNiFi instance
+#nifi.minifi.status.reporter.log.query=instance:health,bulletins
+# The log level at which the status will be logged
+#nifi.minifi.status.reporter.log.level=INFO
+# The period (in milliseconds) at which to log the status
+#nifi.minifi.status.reporter.log.period=60000
+
+# Disable JSR 199 so that we can use JSP's without running a JDK
+java.arg.1=-Dorg.apache.jasper.compiler.disablejsr199=true
+
+# JVM memory settings
+java.arg.2=-Xms256m
+java.arg.3=-Xmx256m
+
+# Enable Remote Debugging
+# java.arg.debug=-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=8000
+
+java.arg.4=-Djava.net.preferIPv4Stack=true
+
+# allowRestrictedHeaders is required for Cluster/Node communications to work properly
+java.arg.5=-Dsun.net.http.allowRestrictedHeaders=true
+java.arg.6=-Djava.protocol.handler.pkgs=sun.net.www.protocol
+
+# The G1GC is still considered experimental but has proven to be very advantageous in providing great
+# performance without significant "stop-the-world" delays.
+#java.arg.13=-XX:+UseG1GC
+
+#Set headless mode by default
+java.arg.14=-Djava.awt.headless=true
+
+java.arg.15=-Djava.security.egd=file:/dev/./urandom

minifi/minifi-integration-tests/src/test/resources/c2/protocol/minifi-edge1/expected.json

@@ -0,0 +1,17 @@
+[
+  {
+    "pattern": "200 OK https://c2-authoritative:10443/c2/config/heartbeat"
+  },
+  {
+    "pattern": "\"operation\":\"UPDATE\",\"operand\":\"CONFIGURATION\""
+  },
+  {
+    "pattern": "200 OK https://c2-authoritative:10443/c2/config\\?class=raspi3"
+  },
+  {
+    "pattern": "200 OK https://c2-authoritative:10443/c2/config/acknowledge"
+  },
+  {
+    "pattern": "^__testTextRaspi3__$"
+  }
+]

minifi/minifi-integration-tests/src/test/resources/c2/protocol/minifi-edge2/bootstrap.conf

@@ -0,0 +1,125 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# C2 Properties
+c2.enable=true
+c2.rest.url=https://c2-authoritative:10443/c2/config/heartbeat
+c2.rest.url.ack=https://c2-authoritative:10443/c2/config/acknowledge
+c2.rest.connectionTimeout=5 sec
+c2.rest.readTimeout=5 sec
+c2.rest.callTimeout=10 sec
+c2.agent.heartbeat.period=2000
+c2.agent.class=raspi3
+c2.config.directory=./conf
+c2.runtime.manifest.identifier=minifi
+c2.runtime.type=minifi-java
+c2.security.truststore.location=./conf/truststore.jks
+c2.security.truststore.password=badTrustPass
+c2.security.truststore.type=JKS
+c2.security.keystore.location=./conf/keystore.jks
+c2.security.keystore.password=badKeystorePass
+c2.security.keystore.type=JKS
+
+# Java command to use when running MiNiFi
+java=java
+
+# Username to use when running MiNiFi. This value will be ignored on Windows.
+run.as=
+
+# Configure where MiNiFi's lib and conf directories live
+lib.dir=./lib
+conf.dir=./conf
+
+# How long to wait after telling MiNiFi to shutdown before explicitly killing the Process
+graceful.shutdown.seconds=20
+
+# The location for the configuration file
+nifi.minifi.config=./conf/config.yml
+
+# Notifiers to use for the associated agent, comma separated list of class names
+nifi.minifi.notifier.ingestors=org.apache.nifi.minifi.bootstrap.configuration.ingestors.FileChangeIngestor
+#nifi.minifi.notifier.ingestors=org.apache.nifi.minifi.bootstrap.configuration.ingestors.RestChangeIngestor
+#nifi.minifi.notifier.ingestors=org.apache.nifi.minifi.bootstrap.configuration.ingestors.PullHttpChangeIngestor
+
+# File change notifier configuration
+
+# Path of the file to monitor for changes.  When these occur, the FileChangeNotifier, if configured, will begin the configuration reloading process
+nifi.minifi.notifier.ingestors.file.config.path=./conf/config-new.yml
+# How frequently the file specified by 'nifi.minifi.notifier.file.config.path' should be evaluated for changes.
+nifi.minifi.notifier.ingestors.file.polling.period.seconds=2
+
+# Rest change notifier configuration
+
+# Port on which the Jetty server will bind to, keep commented for a random open port
+#nifi.minifi.notifier.ingestors.receive.http.port=8338
+
+#Pull HTTP change notifier configuration
+
+# Hostname on which to pull configurations from
+# nifi.minifi.notifier.ingestors.pull.http.hostname=c2-authoritative
+# Port on which to pull configurations from
+# nifi.minifi.notifier.ingestors.pull.http.port=10443
+# Path to pull configurations from
+# nifi.minifi.notifier.ingestors.pull.http.path=/c2/config
+# Query string to pull configurations with
+# nifi.minifi.notifier.ingestors.pull.http.query=net=edge1&class=raspi3
+# Period on which to pull configurations from, defaults to 5 minutes if commented out
+# nifi.minifi.notifier.ingestors.pull.http.period.ms=3000
+
+# nifi.minifi.notifier.ingestors.pull.http.keystore.location=./conf/keystore.jks
+# nifi.minifi.notifier.ingestors.pull.http.keystore.type=JKS
+# nifi.minifi.notifier.ingestors.pull.http.keystore.password=badKeystorePass
+# nifi.minifi.notifier.ingestors.pull.http.truststore.location=./conf/truststore.jks
+# nifi.minifi.notifier.ingestors.pull.http.truststore.type=JKS
+# nifi.minifi.notifier.ingestors.pull.http.truststore.password=badTrustPass
+
+# Periodic Status Reporters to use for the associated agent, comma separated list of class names
+#nifi.minifi.status.reporter.components=org.apache.nifi.minifi.bootstrap.status.reporters.StatusLogger
+
+# Periodic Status Logger configuration
+
+# The FlowStatus query to submit to the MiNiFi instance
+#nifi.minifi.status.reporter.log.query=instance:health,bulletins
+# The log level at which the status will be logged
+#nifi.minifi.status.reporter.log.level=INFO
+# The period (in milliseconds) at which to log the status
+#nifi.minifi.status.reporter.log.period=60000
+
+# Disable JSR 199 so that we can use JSP's without running a JDK
+java.arg.1=-Dorg.apache.jasper.compiler.disablejsr199=true
+
+# JVM memory settings
+java.arg.2=-Xms256m
+java.arg.3=-Xmx256m
+
+# Enable Remote Debugging
+# java.arg.debug=-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=8000
+
+java.arg.4=-Djava.net.preferIPv4Stack=true
+
+# allowRestrictedHeaders is required for Cluster/Node communications to work properly
+java.arg.5=-Dsun.net.http.allowRestrictedHeaders=true
+java.arg.6=-Djava.protocol.handler.pkgs=sun.net.www.protocol
+
+# The G1GC is still considered experimental but has proven to be very advantageous in providing great
+# performance without significant "stop-the-world" delays.
+#java.arg.13=-XX:+UseG1GC
+
+#Set headless mode by default
+java.arg.14=-Djava.awt.headless=true
+
+java.arg.15=-Djava.security.egd=file:/dev/./urandom

minifi/minifi-integration-tests/src/test/resources/c2/protocol/minifi-edge2/expected.json

@@ -0,0 +1,17 @@
+[
+  {
+    "pattern": "200 OK https://c2-authoritative:10443/c2/config/heartbeat"
+  },
+  {
+    "pattern": "\"operation\":\"UPDATE\",\"operand\":\"CONFIGURATION\""
+  },
+  {
+    "pattern": "200 OK https://c2-authoritative:10443/c2/config\\?class=raspi3"
+  },
+  {
+    "pattern": "200 OK https://c2-authoritative:10443/c2/config/acknowledge"
+  },
+  {
+    "pattern": "^__testTextRaspi3__$"
+  }
+]

minifi/minifi-integration-tests/src/test/resources/c2/protocol/minifi-edge3/bootstrap.conf

@@ -0,0 +1,125 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# C2 Properties
+c2.enable=true
+c2.rest.url=https://c2-authoritative:10443/c2/config/heartbeat
+c2.rest.url.ack=https://c2-authoritative:10443/c2/config/acknowledge
+c2.rest.connectionTimeout=5 sec
+c2.rest.readTimeout=5 sec
+c2.rest.callTimeout=10 sec
+c2.agent.heartbeat.period=2000
+c2.agent.class=raspi4
+c2.config.directory=./conf
+c2.runtime.manifest.identifier=minifi
+c2.runtime.type=minifi-java
+c2.security.truststore.location=./conf/truststore.jks
+c2.security.truststore.password=badTrustPass
+c2.security.truststore.type=JKS
+c2.security.keystore.location=./conf/keystore.jks
+c2.security.keystore.password=badKeystorePass
+c2.security.keystore.type=JKS
+
+# Java command to use when running MiNiFi
+java=java
+
+# Username to use when running MiNiFi. This value will be ignored on Windows.
+run.as=
+
+# Configure where MiNiFi's lib and conf directories live
+lib.dir=./lib
+conf.dir=./conf
+
+# How long to wait after telling MiNiFi to shutdown before explicitly killing the Process
+graceful.shutdown.seconds=20
+
+# The location for the configuration file
+nifi.minifi.config=./conf/config.yml
+
+# Notifiers to use for the associated agent, comma separated list of class names
+nifi.minifi.notifier.ingestors=org.apache.nifi.minifi.bootstrap.configuration.ingestors.FileChangeIngestor
+#nifi.minifi.notifier.ingestors=org.apache.nifi.minifi.bootstrap.configuration.ingestors.RestChangeIngestor
+#nifi.minifi.notifier.ingestors=org.apache.nifi.minifi.bootstrap.configuration.ingestors.PullHttpChangeIngestor
+
+# File change notifier configuration
+
+# Path of the file to monitor for changes.  When these occur, the FileChangeNotifier, if configured, will begin the configuration reloading process
+nifi.minifi.notifier.ingestors.file.config.path=./conf/config-new.yml
+# How frequently the file specified by 'nifi.minifi.notifier.file.config.path' should be evaluated for changes.
+nifi.minifi.notifier.ingestors.file.polling.period.seconds=2
+
+# Rest change notifier configuration
+
+# Port on which the Jetty server will bind to, keep commented for a random open port
+#nifi.minifi.notifier.ingestors.receive.http.port=8338
+
+#Pull HTTP change notifier configuration
+
+# Hostname on which to pull configurations from
+# nifi.minifi.notifier.ingestors.pull.http.hostname=c2-authoritative
+# Port on which to pull configurations from
+# nifi.minifi.notifier.ingestors.pull.http.port=10443
+# Path to pull configurations from
+# nifi.minifi.notifier.ingestors.pull.http.path=/c2/config
+# Query string to pull configurations with
+# nifi.minifi.notifier.ingestors.pull.http.query=net=edge1&class=raspi4
+# Period on which to pull configurations from, defaults to 5 minutes if commented out
+# nifi.minifi.notifier.ingestors.pull.http.period.ms=3000
+
+# nifi.minifi.notifier.ingestors.pull.http.keystore.location=./conf/keystore.jks
+# nifi.minifi.notifier.ingestors.pull.http.keystore.type=JKS
+# nifi.minifi.notifier.ingestors.pull.http.keystore.password=badKeystorePass
+# nifi.minifi.notifier.ingestors.pull.http.truststore.location=./conf/truststore.jks
+# nifi.minifi.notifier.ingestors.pull.http.truststore.type=JKS
+# nifi.minifi.notifier.ingestors.pull.http.truststore.password=badTrustPass
+
+# Periodic Status Reporters to use for the associated agent, comma separated list of class names
+#nifi.minifi.status.reporter.components=org.apache.nifi.minifi.bootstrap.status.reporters.StatusLogger
+
+# Periodic Status Logger configuration
+
+# The FlowStatus query to submit to the MiNiFi instance
+#nifi.minifi.status.reporter.log.query=instance:health,bulletins
+# The log level at which the status will be logged
+#nifi.minifi.status.reporter.log.level=INFO
+# The period (in milliseconds) at which to log the status
+#nifi.minifi.status.reporter.log.period=60000
+
+# Disable JSR 199 so that we can use JSP's without running a JDK
+java.arg.1=-Dorg.apache.jasper.compiler.disablejsr199=true
+
+# JVM memory settings
+java.arg.2=-Xms256m
+java.arg.3=-Xmx256m
+
+# Enable Remote Debugging
+# java.arg.debug=-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=8000
+
+java.arg.4=-Djava.net.preferIPv4Stack=true
+
+# allowRestrictedHeaders is required for Cluster/Node communications to work properly
+java.arg.5=-Dsun.net.http.allowRestrictedHeaders=true
+java.arg.6=-Djava.protocol.handler.pkgs=sun.net.www.protocol
+
+# The G1GC is still considered experimental but has proven to be very advantageous in providing great
+# performance without significant "stop-the-world" delays.
+#java.arg.13=-XX:+UseG1GC
+
+#Set headless mode by default
+java.arg.14=-Djava.awt.headless=true
+
+java.arg.15=-Djava.security.egd=file:/dev/./urandom

minifi/minifi-integration-tests/src/test/resources/c2/protocol/minifi-edge3/expected.json

@@ -0,0 +1,17 @@
+[
+  {
+    "pattern": "200 OK https://c2-authoritative:10443/c2/config/heartbeat"
+  },
+  {
+    "pattern": "\"operation\":\"UPDATE\",\"operand\":\"CONFIGURATION\""
+  },
+  {
+    "pattern": "200 OK https://c2-authoritative:10443/c2/config\\?class=raspi4"
+  },
+  {
+    "pattern": "200 OK https://c2-authoritative:10443/c2/config/acknowledge"
+  },
+  {
+    "pattern": "^__testTextRaspi4__$"
+  }
+]

minifi/minifi-integration-tests/src/test/resources/docker-compose-c2-protocol.yml

@@ -0,0 +1,100 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the \"License\"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an \"AS IS\" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: "2"
+
+services:
+  c2-authoritative:
+    build:
+      context: ./
+      dockerfile: Dockerfile.minific2.test
+    image: apacheminific2-test
+    ports:
+      - "10443"
+    hostname: c2-authoritative
+    volumes:
+      - ./c2/protocol/c2-authoritative/files:/opt/minifi-c2/minifi-c2-${minifi.c2.version}/files
+      - ./c2/protocol/c2-authoritative/conf/minifi-c2-context.xml:/opt/minifi-c2/minifi-c2-${minifi.c2.version}/conf/minifi-c2-context.xml
+      - ./c2/protocol/c2-authoritative/conf/c2.properties:/opt/minifi-c2/minifi-c2-${minifi.c2.version}/conf/c2.properties
+      - ./c2/protocol/c2-authoritative/conf/authorities.yaml:/opt/minifi-c2/minifi-c2-${minifi.c2.version}/conf/authorities.yaml
+      - ./c2/protocol/c2-authoritative/conf/authorizations.yaml:/opt/minifi-c2/minifi-c2-${minifi.c2.version}/conf/authorizations.yaml
+      - ./certificates-c2-protocol/c2-authoritative/keystore.jks:/opt/minifi-c2/minifi-c2-${minifi.c2.version}/conf/keystore.jks
+      - ./certificates-c2-protocol/c2-authoritative/truststore.jks:/opt/minifi-c2/minifi-c2-${minifi.c2.version}/conf/truststore.jks
+    networks:
+      - edge
+
+  minifi-edge1:
+    build:
+      context: ./
+      dockerfile: ./Dockerfile.minifi.test
+    image: apacheminifi-test
+    ports:
+      - "8000"
+    volumes:
+      - ./tailFileServer.py:/home/minifi/tailFileServer.py
+      - ./c2/protocol/minifi-edge1/bootstrap.conf:/opt/minifi/minifi-${minifi.version}/conf/bootstrap.conf
+      - ./logback.xml:/opt/minifi/minifi-${minifi.version}/conf/logback.xml
+      - ./certificates-c2-protocol/minifi-edge1/keystore.jks:/opt/minifi/minifi-${minifi.version}/conf/keystore.jks
+      - ./certificates-c2-protocol/minifi-edge1/truststore.jks:/opt/minifi/minifi-${minifi.version}/conf/truststore.jks
+    entrypoint:
+      - sh
+      - -c
+      - /opt/minifi/minifi-${minifi.version}/bin/minifi.sh start && python /home/minifi/tailFileServer.py --file /opt/minifi/minifi-${minifi.version}/logs/minifi-app.log
+    networks:
+      - edge
+
+  minifi-edge2:
+    build:
+      context: ./
+      dockerfile: ./Dockerfile.minifi.test
+    image: apacheminifi-test
+    ports:
+      - "8000"
+    volumes:
+      - ./tailFileServer.py:/home/minifi/tailFileServer.py
+      - ./c2/protocol/minifi-edge2/bootstrap.conf:/opt/minifi/minifi-${minifi.version}/conf/bootstrap.conf
+      - ./logback.xml:/opt/minifi/minifi-${minifi.version}/conf/logback.xml
+      - ./certificates-c2-protocol/minifi-edge2/keystore.jks:/opt/minifi/minifi-${minifi.version}/conf/keystore.jks
+      - ./certificates-c2-protocol/minifi-edge2/truststore.jks:/opt/minifi/minifi-${minifi.version}/conf/truststore.jks
+    entrypoint:
+      - sh
+      - -c
+      - /opt/minifi/minifi-${minifi.version}/bin/minifi.sh start && python /home/minifi/tailFileServer.py --file /opt/minifi/minifi-${minifi.version}/logs/minifi-app.log
+    networks:
+      - edge
+
+  minifi-edge3:
+    build:
+      context: ./
+      dockerfile: ./Dockerfile.minifi.test
+    image: apacheminifi-test
+    ports:
+      - "8000"
+    volumes:
+      - ./tailFileServer.py:/home/minifi/tailFileServer.py
+      - ./c2/protocol/minifi-edge3/bootstrap.conf:/opt/minifi/minifi-${minifi.version}/conf/bootstrap.conf
+      - ./logback.xml:/opt/minifi/minifi-${minifi.version}/conf/logback.xml
+      - ./certificates-c2-protocol/minifi-edge3/keystore.jks:/opt/minifi/minifi-${minifi.version}/conf/keystore.jks
+      - ./certificates-c2-protocol/minifi-edge3/truststore.jks:/opt/minifi/minifi-${minifi.version}/conf/truststore.jks
+    entrypoint:
+      - sh
+      - -c
+      - /opt/minifi/minifi-${minifi.version}/bin/minifi.sh start && python /home/minifi/tailFileServer.py --file /opt/minifi/minifi-${minifi.version}/logs/minifi-app.log
+    networks:
+      - edge
+
+networks:
+  edge:
+    driver: bridge

minifi/minifi-integration-tests/src/test/resources/logback.xml

@@ -56,6 +56,7 @@
     <!-- valid logging levels: TRACE, DEBUG, INFO, WARN, ERROR -->
     
     <logger name="org.apache.nifi" level="INFO"/>
+    <logger name="org.apache.nifi.c2.client.http.C2HttpClient" level="DEBUG"/>
     <logger name="org.apache.nifi.processors" level="WARN"/>
     <logger name="org.apache.nifi.processors.standard.LogAttribute" level="INFO"/>
     <logger name="org.apache.nifi.controller.repository.StandardProcessSession" level="DEBUG" />
@@ -77,6 +78,7 @@
         Logger for capturing Bootstrap logs and MiNiFi's standard error and standard out.
     -->
     <logger name="org.apache.nifi.minifi.bootstrap" level="INFO" additivity="false">
+        <appender-ref ref="CONSOLE" />
         <appender-ref ref="BOOTSTRAP_FILE" />
     </logger>
     <logger name="org.apache.nifi.minifi.bootstrap.Command" level="INFO" additivity="false">
@@ -86,17 +88,19 @@
 
     <!-- Everything written to MiNiFi's Standard Out will be logged with the logger org.apache.nifi.minifi.StdOut at INFO level -->
     <logger name="org.apache.nifi.minifi.StdOut" level="INFO" additivity="false">
+        <appender-ref ref="CONSOLE" />
         <appender-ref ref="BOOTSTRAP_FILE" />
     </logger>
 
     <!-- Everything written to MiNiFi's Standard Error will be logged with the logger org.apache.nifi.minifi.StdErr at ERROR level -->
-	<logger name="org.apache.nifi.minifi.StdErr" level="ERROR" additivity="false">
-    	<appender-ref ref="BOOTSTRAP_FILE" />
+    <logger name="org.apache.nifi.minifi.StdErr" level="ERROR" additivity="false">
+        <appender-ref ref="CONSOLE" />
+        <appender-ref ref="BOOTSTRAP_FILE" />
     </logger>
 
-
-    <root level="INFO">
+    <root level="DEBUG">
         <appender-ref ref="APP_FILE"/>
+        <appender-ref ref="CONSOLE"/>
     </root>
     
 </configuration>