diff --git a/c2/c2-client-bundle/README.md b/c2/c2-client-bundle/README.md new file mode 100644 index 0000000000..90915076d5 --- /dev/null +++ b/c2/c2-client-bundle/README.md @@ -0,0 +1,61 @@ + +## Apache NiFi MiNiFi Command and Control (C2) Client +The c2-client-bundle provides implementation for the client aspect of the [C2 Protocol](https://cwiki.apache.org/confluence/display/MINIFI/C2+Design). The essence of the implementation is the heartbeat construction and the communication with the [C2 server](../../../../minifi-c2/README.md) via the C2HttpClient. + +Currently, relying on the [C2 Protocol API](../c2-protocol) is limited to sending heartbeats and processing/acknowledging UPDATE configuration operation in the response (if any). When exposed the new configuration will be downloaded and passed back to the system using the C2 protocol. + +When C2 is enabled, C2ClientService will be scheduled to send heartbeats periodically, so the C2 Server can notify the client about any operations that is defined by the protocol and needs to be executed on the client side. + +Using the client means that configuration changes and other operations can be triggered and controlled centrally via the C2 server making the management of clients more simple and configuring them more flexible. The client supports bidirectional TLS authentication. + +### Configuration +To use the client, the parameters coming from `C2ClientConfig` need to be properly set (this configuration class is also used for instantiating `C2HeartbeatFactory` and `C2HttpClient`) + +``` + # The C2 Server endpoint where the heartbeat is sent + private final String c2Url; + + # The C2 Server endpoint where the acknowledge is sent + private final String c2AckUrl; + + # The class the agent belongs to (flow definition is tied to agent class on the server side) + private final String agentClass; + + # Unique identifier for the agent if not provided it will be generated + private final String agentIdentifier; + + # Directory where persistent configuration (e.g.: generated agent and device id will be persisted) + private final String confDirectory; + + # Property of RuntimeManifest defined in c2-protocol. A unique identifier for the manifest + private final String runtimeManifestIdentifier; + + # Property of RuntimeManifest defined in c2-protocol. The type of the runtime binary. Usually set when the runtime is built + private final String runtimeType; + + # The frequency of sending the heartbeats. This property is used by the c2-client-bundle user who should schedule the client + private final Long heartbeatPeriod; + + # Security properties for communication with the C2 Server + private final String keystoreFilename; + private final String keystorePass; + private final String keyPass; + private final KeystoreType keystoreType; + private final String truststoreFilename; + private final String truststorePass; + private final KeystoreType truststoreType; + private final HostnameVerifier hostnameVerifier; +``` \ No newline at end of file diff --git a/c2/c2-client-bundle/c2-client-api/pom.xml b/c2/c2-client-bundle/c2-client-api/pom.xml new file mode 100644 index 0000000000..4a9dca9626 --- /dev/null +++ b/c2/c2-client-bundle/c2-client-api/pom.xml @@ -0,0 +1,37 @@ + + + + 4.0.0 + + + c2-client-bundle + org.apache.nifi + 1.17.0-SNAPSHOT + + + c2-client-api + jar + + + + org.apache.nifi + c2-protocol-api + 1.17.0-SNAPSHOT + + + diff --git a/c2/c2-client-bundle/c2-client-api/src/main/java/org/apache/nifi/c2/client/api/C2Client.java b/c2/c2-client-bundle/c2-client-api/src/main/java/org/apache/nifi/c2/client/api/C2Client.java new file mode 100644 index 0000000000..e9f61e1e94 --- /dev/null +++ b/c2/c2-client-bundle/c2-client-api/src/main/java/org/apache/nifi/c2/client/api/C2Client.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.c2.client.api; + +import java.util.Optional; +import org.apache.nifi.c2.protocol.api.C2Heartbeat; +import org.apache.nifi.c2.protocol.api.C2HeartbeatResponse; +import org.apache.nifi.c2.protocol.api.C2OperationAck; + +/** + * Defines interface methods used to implement a C2 Client. The controller can be application-specific but is used for such tasks as updating the flow. + */ +public interface C2Client { + + /** + * Responsible for sending the C2Heartbeat to the C2 Server + * + * @param heartbeat the heartbeat to be sent + * @return optional response from the C2 Server if the response arrived it will be populated + */ + Optional publishHeartbeat(C2Heartbeat heartbeat); + + /** + * Retrive the content of the new flow from the C2 Server + * + * @param flowUpdateUrl url where the content should be downloaded from + * @return the actual downloaded content. Will be empty if no content can be downloaded + */ + Optional retrieveUpdateContent(String flowUpdateUrl); + + /** + * After operation completed the acknowledge to be sent to the C2 Server + * + * @param operationAck the acknowledge details to be sent + */ + void acknowledgeOperation(C2OperationAck operationAck); +} diff --git a/c2/c2-client-bundle/c2-client-api/src/main/java/org/apache/nifi/c2/client/api/C2Serializer.java b/c2/c2-client-bundle/c2-client-api/src/main/java/org/apache/nifi/c2/client/api/C2Serializer.java new file mode 100644 index 0000000000..6f98a09fd5 --- /dev/null +++ b/c2/c2-client-bundle/c2-client-api/src/main/java/org/apache/nifi/c2/client/api/C2Serializer.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.c2.client.api; + +import java.util.Optional; + +/** + * Helper class to support central configuration and functionality for serialisation / deserialisation + */ +public interface C2Serializer { + + /** + * Helper to serialise object + * + * @param content object to be serialised + * @param the type of the object + * @return the serialised string representation of the parameter object if it was successful empty otherwise + */ + Optional serialize(T content); + + /** + * Helper to deserialise an object + * + * @param content the string representation of the object to be deserialsed + * @param valueType the class of the target object + * @param the type of the target object + * @return the deserialised object if successful empty otherwise + */ + Optional deserialize(String content, Class valueType); +} diff --git a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/ConfigurationFileHolder.java b/c2/c2-client-bundle/c2-client-api/src/main/java/org/apache/nifi/c2/client/api/ConfigurationFileHolder.java similarity index 81% rename from minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/ConfigurationFileHolder.java rename to c2/c2-client-bundle/c2-client-api/src/main/java/org/apache/nifi/c2/client/api/ConfigurationFileHolder.java index d5113e3597..ab43ff79b8 100644 --- a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/ConfigurationFileHolder.java +++ b/c2/c2-client-bundle/c2-client-api/src/main/java/org/apache/nifi/c2/client/api/ConfigurationFileHolder.java @@ -15,12 +15,20 @@ * limitations under the License. */ -package org.apache.nifi.minifi.bootstrap; +package org.apache.nifi.c2.client.api; import java.nio.ByteBuffer; import java.util.concurrent.atomic.AtomicReference; +/** + * Should be implemented by the class which bootstraps the agent. + */ public interface ConfigurationFileHolder { + /** + * Retrieve the reference to the config file + * + * @return config file reference + */ AtomicReference getConfigFileReference(); } diff --git a/c2/c2-client-bundle/c2-client-api/src/main/java/org/apache/nifi/c2/client/api/Differentiator.java b/c2/c2-client-bundle/c2-client-api/src/main/java/org/apache/nifi/c2/client/api/Differentiator.java new file mode 100644 index 0000000000..7f529becbc --- /dev/null +++ b/c2/c2-client-bundle/c2-client-api/src/main/java/org/apache/nifi/c2/client/api/Differentiator.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.c2.client.api; + +import java.io.IOException; +import java.util.Properties; + +/** + * Helper to support differentiating between config files to recognise changes + * + * @param the type of the config files + */ +public interface Differentiator { + + /** + * Initialise the differentiator with the initial configuration + * + * @param properties the properties to be used + * @param configurationFileHolder holder for the config file + */ + void initialize(Properties properties, ConfigurationFileHolder configurationFileHolder); + + /** + * Determine whether the config file changed + * + * @param input the conetnt of the new config file + * @return true if changed and false if not + * @throws IOException when there is a config file reading related error + */ + boolean isNew(T input) throws IOException; +} diff --git a/c2/c2-client-bundle/c2-client-api/src/main/java/org/apache/nifi/c2/client/api/IdGenerator.java b/c2/c2-client-bundle/c2-client-api/src/main/java/org/apache/nifi/c2/client/api/IdGenerator.java new file mode 100644 index 0000000000..ca63df4bac --- /dev/null +++ b/c2/c2-client-bundle/c2-client-api/src/main/java/org/apache/nifi/c2/client/api/IdGenerator.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.c2.client.api; + +/** + * Id generator to be used in case user is not providing an optional id + */ +public interface IdGenerator { + + /** + * Generate a random id + * + * @return the generated id + */ + String generate(); +} diff --git a/c2/c2-client-bundle/c2-client-base/pom.xml b/c2/c2-client-bundle/c2-client-base/pom.xml new file mode 100644 index 0000000000..1e9464ecd5 --- /dev/null +++ b/c2/c2-client-bundle/c2-client-base/pom.xml @@ -0,0 +1,55 @@ + + + + 4.0.0 + + + c2-client-bundle + org.apache.nifi + 1.17.0-SNAPSHOT + + + c2-client-base + jar + + + + org.apache.nifi + c2-client-api + 1.17.0-SNAPSHOT + provided + + + org.apache.nifi + c2-protocol-api + 1.17.0-SNAPSHOT + + + org.apache.commons + commons-lang3 + + + com.fasterxml.jackson.core + jackson-databind + + + com.fasterxml.jackson.core + jackson-annotations + + + diff --git a/c2/c2-client-bundle/c2-client-base/src/main/java/org/apache/nifi/c2/client/C2ClientConfig.java b/c2/c2-client-bundle/c2-client-base/src/main/java/org/apache/nifi/c2/client/C2ClientConfig.java new file mode 100644 index 0000000000..5d7481dd9d --- /dev/null +++ b/c2/c2-client-bundle/c2-client-base/src/main/java/org/apache/nifi/c2/client/C2ClientConfig.java @@ -0,0 +1,255 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.c2.client; + +/** + * Configuration for a C2 Client. + */ +public class C2ClientConfig { + + private final String c2Url; + private final String c2AckUrl; + private final String agentClass; + private final String agentIdentifier; + private final String confDirectory; + private final String runtimeManifestIdentifier; + private final String runtimeType; + private final long heartbeatPeriod; + private final String keystoreFilename; + private final String keystorePass; + private final String keyPass; + private final String keystoreType; + private final String truststoreFilename; + private final String truststorePass; + private final String truststoreType; + private final long callTimeout; + private final long readTimeout; + private final long connectTimeout; + + + private C2ClientConfig(final Builder builder) { + this.c2Url = builder.c2Url; + this.c2AckUrl = builder.c2AckUrl; + this.agentClass = builder.agentClass; + this.agentIdentifier = builder.agentIdentifier; + this.confDirectory = builder.confDirectory; + this.runtimeManifestIdentifier = builder.runtimeManifestIdentifier; + this.runtimeType = builder.runtimeType; + this.heartbeatPeriod = builder.heartbeatPeriod; + this.callTimeout = builder.callTimeout; + this.keystoreFilename = builder.keystoreFilename; + this.keystorePass = builder.keystorePass; + this.keyPass = builder.keyPass; + this.keystoreType = builder.keystoreType; + this.truststoreFilename = builder.truststoreFilename; + this.truststorePass = builder.truststorePass; + this.truststoreType = builder.truststoreType; + this.readTimeout = builder.readTimeout; + this.connectTimeout = builder.connectTimeout; + } + + public String getC2Url() { + return c2Url; + } + + public String getC2AckUrl() { + return c2AckUrl; + } + + public String getAgentClass() { + return agentClass; + } + + public String getAgentIdentifier() { + return agentIdentifier; + } + + public String getConfDirectory() { + return confDirectory; + } + + public String getRuntimeManifestIdentifier() { + return runtimeManifestIdentifier; + } + + public String getRuntimeType() { + return runtimeType; + } + + public long getHeartbeatPeriod() { + return heartbeatPeriod; + } + + public long getCallTimeout() { + return callTimeout; + } + + public String getKeystoreFilename() { + return keystoreFilename; + } + + public String getKeystorePass() { + return keystorePass; + } + + public String getKeyPass() { + return keyPass; + } + + public String getKeystoreType() { + return keystoreType; + } + + public String getTruststoreFilename() { + return truststoreFilename; + } + + public String getTruststorePass() { + return truststorePass; + } + + public String getTruststoreType() { + return truststoreType; + } + + public long getReadTimeout() { + return readTimeout; + } + + public long getConnectTimeout() { + return connectTimeout; + } + + /** + * Builder for client configuration. + */ + public static class Builder { + + private String c2Url; + private String c2AckUrl; + private String agentClass; + private String agentIdentifier; + private String confDirectory; + private String runtimeManifestIdentifier; + private String runtimeType; + private long heartbeatPeriod; + private long callTimeout; + private String keystoreFilename; + private String keystorePass; + private String keyPass; + private String keystoreType; + private String truststoreFilename; + private String truststorePass; + private String truststoreType; + private long readTimeout; + private long connectTimeout; + + public Builder c2Url(final String c2Url) { + this.c2Url = c2Url; + return this; + } + + public Builder c2AckUrl(final String c2AckUrl) { + this.c2AckUrl = c2AckUrl; + return this; + } + + public Builder agentClass(final String agentClass) { + this.agentClass = agentClass; + return this; + } + + public Builder agentIdentifier(final String agentIdentifier) { + this.agentIdentifier = agentIdentifier; + return this; + } + + public Builder confDirectory(final String confDirectory) { + this.confDirectory = confDirectory; + return this; + } + + public Builder runtimeManifestIdentifier(final String runtimeManifestIdentifier) { + this.runtimeManifestIdentifier = runtimeManifestIdentifier; + return this; + } + + public Builder runtimeType(final String runtimeType) { + this.runtimeType = runtimeType; + return this; + } + + public Builder heartbeatPeriod(final long heartbeatPeriod) { + this.heartbeatPeriod = heartbeatPeriod; + return this; + } + + public Builder callTimeout(final long callTimeout) { + this.callTimeout = callTimeout; + return this; + } + + public Builder keystoreFilename(final String keystoreFilename) { + this.keystoreFilename = keystoreFilename; + return this; + } + + public Builder keystorePassword(final String keystorePass) { + this.keystorePass = keystorePass; + return this; + } + + public Builder keyPassword(final String keyPass) { + this.keyPass = keyPass; + return this; + } + + public Builder keystoreType(final String keystoreType) { + this.keystoreType = keystoreType; + return this; + } + + public Builder truststoreFilename(final String truststoreFilename) { + this.truststoreFilename = truststoreFilename; + return this; + } + + public Builder truststorePassword(final String truststorePass) { + this.truststorePass = truststorePass; + return this; + } + + public Builder truststoreType(final String truststoreType) { + this.truststoreType = truststoreType; + return this; + } + + public Builder readTimeout(final long readTimeout) { + this.readTimeout = readTimeout; + return this; + } + + public Builder connectTimeout(final long connectTimeout) { + this.connectTimeout = connectTimeout; + return this; + } + + public C2ClientConfig build() { + return new C2ClientConfig(this); + } + } +} diff --git a/c2/c2-client-bundle/c2-client-base/src/main/java/org/apache/nifi/c2/client/PersistentUuidGenerator.java b/c2/c2-client-bundle/c2-client-base/src/main/java/org/apache/nifi/c2/client/PersistentUuidGenerator.java new file mode 100644 index 0000000000..9c645180e1 --- /dev/null +++ b/c2/c2-client-bundle/c2-client-base/src/main/java/org/apache/nifi/c2/client/PersistentUuidGenerator.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.c2.client; + +import org.apache.nifi.c2.client.api.IdGenerator; +import org.apache.nifi.c2.util.FileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.Arrays; +import java.util.List; +import java.util.UUID; + +public class PersistentUuidGenerator implements IdGenerator { + + private static final Logger logger = LoggerFactory.getLogger(PersistentUuidGenerator.class); + + private final File persistenceLocation; + + public PersistentUuidGenerator(final File persistenceLocation) { + this.persistenceLocation = persistenceLocation; + } + + @Override + public String generate() { + if (this.persistenceLocation.exists()) { + return readFile(); + } else { + return makeFile(); + } + } + + private String readFile() { + try { + final List fileLines = Files.readAllLines(persistenceLocation.toPath()); + if (fileLines.size() != 1) { + throw new IllegalStateException(String.format("The file %s for the persisted identifier has the incorrect format.", persistenceLocation)); + } + final String uuid = fileLines.get(0); + return uuid; + } catch (IOException e) { + throw new IllegalStateException(String.format("Could not read file %s for persisted identifier.", persistenceLocation), e); + + } + } + + private String makeFile() { + try { + final File parentDirectory = persistenceLocation.getParentFile(); + FileUtils.ensureDirectoryExistAndCanAccess(parentDirectory); + final String uuid = UUID.randomUUID().toString(); + Files.write(persistenceLocation.toPath(), Arrays.asList(uuid)); + logger.debug("Created identifier {} at {}", uuid, persistenceLocation); + return uuid; + } catch (IOException e) { + throw new IllegalStateException(String.format("Could not create file %s as persistence file.", persistenceLocation), e); + } + } +} diff --git a/c2/c2-client-bundle/c2-client-base/src/main/java/org/apache/nifi/c2/serializer/C2JacksonSerializer.java b/c2/c2-client-bundle/c2-client-base/src/main/java/org/apache/nifi/c2/serializer/C2JacksonSerializer.java new file mode 100644 index 0000000000..1d8317d787 --- /dev/null +++ b/c2/c2-client-bundle/c2-client-base/src/main/java/org/apache/nifi/c2/serializer/C2JacksonSerializer.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.c2.serializer; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.Optional; +import org.apache.nifi.c2.client.api.C2Serializer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class C2JacksonSerializer implements C2Serializer { + + private static final Logger logger = LoggerFactory.getLogger(C2JacksonSerializer.class); + + private final ObjectMapper objectMapper; + + public C2JacksonSerializer() { + objectMapper = new ObjectMapper(); + objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + objectMapper.setSerializationInclusion(JsonInclude.Include.NON_NULL); + objectMapper.enable(DeserializationFeature.READ_ENUMS_USING_TO_STRING); + } + + @Override + public Optional serialize(T object) { + if (object == null) { + logger.trace("C2 Object was null. Nothing to serialize. Returning empty."); + return Optional.empty(); + } + + String contentString = null; + try { + contentString = objectMapper.writeValueAsString(object); + } catch (JsonProcessingException e) { + logger.error("Object serialization to JSON failed", e); + } + + return Optional.ofNullable(contentString); + } + + @Override + public Optional deserialize(String content, Class valueType) { + if (content == null) { + logger.trace("Content for deserialization was null. Returning empty."); + return Optional.empty(); + } + + T responseObject = null; + try { + responseObject = objectMapper.readValue(content, valueType); + } catch (JsonProcessingException e) { + logger.error("Object deserialization from JSON failed", e); + } + + return Optional.ofNullable(responseObject); + } +} diff --git a/c2/c2-client-bundle/c2-client-base/src/main/java/org/apache/nifi/c2/util/FileUtils.java b/c2/c2-client-bundle/c2-client-base/src/main/java/org/apache/nifi/c2/util/FileUtils.java new file mode 100644 index 0000000000..fbab93cb64 --- /dev/null +++ b/c2/c2-client-bundle/c2-client-base/src/main/java/org/apache/nifi/c2/util/FileUtils.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.c2.util; + +import java.io.File; +import java.io.IOException; + +public class FileUtils { + + public static void ensureDirectoryExistAndCanAccess(final File dir) throws IOException { + if (dir.exists() && !dir.isDirectory()) { + throw new IOException(dir.getAbsolutePath() + " is not a directory"); + } else if (!dir.exists()) { + final boolean made = dir.mkdirs(); + if (!made) { + throw new IOException(dir.getAbsolutePath() + " could not be created"); + } + } + if (!(dir.canRead() && dir.canWrite())) { + throw new IOException(dir.getAbsolutePath() + " directory does not have read/write privilege"); + } + } +} diff --git a/c2/c2-client-bundle/c2-client-http/pom.xml b/c2/c2-client-bundle/c2-client-http/pom.xml new file mode 100644 index 0000000000..75076c051e --- /dev/null +++ b/c2/c2-client-bundle/c2-client-http/pom.xml @@ -0,0 +1,51 @@ + + + + 4.0.0 + + + c2-client-bundle + org.apache.nifi + 1.17.0-SNAPSHOT + + + c2-client-http + jar + + + + org.apache.nifi + c2-client-api + 1.17.0-SNAPSHOT + provided + + + org.apache.nifi + c2-client-base + 1.17.0-SNAPSHOT + + + com.squareup.okhttp3 + okhttp + + + com.squareup.okhttp3 + logging-interceptor + + + diff --git a/c2/c2-client-bundle/c2-client-http/src/main/java/org/apache/nifi/c2/client/http/C2HttpClient.java b/c2/c2-client-bundle/c2-client-http/src/main/java/org/apache/nifi/c2/client/http/C2HttpClient.java new file mode 100644 index 0000000000..7f9b1412af --- /dev/null +++ b/c2/c2-client-bundle/c2-client-http/src/main/java/org/apache/nifi/c2/client/http/C2HttpClient.java @@ -0,0 +1,248 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.c2.client.http; + +import java.io.FileInputStream; +import java.io.IOException; +import java.security.KeyStore; +import java.security.NoSuchAlgorithmException; +import java.util.Optional; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import javax.net.ssl.KeyManagerFactory; +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLSocketFactory; +import javax.net.ssl.TrustManager; +import javax.net.ssl.TrustManagerFactory; +import javax.net.ssl.X509TrustManager; +import okhttp3.MediaType; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.RequestBody; +import okhttp3.Response; +import okhttp3.ResponseBody; +import okhttp3.logging.HttpLoggingInterceptor; +import org.apache.commons.lang3.StringUtils; +import org.apache.nifi.c2.client.C2ClientConfig; +import org.apache.nifi.c2.client.api.C2Client; +import org.apache.nifi.c2.client.api.C2Serializer; +import org.apache.nifi.c2.protocol.api.C2Heartbeat; +import org.apache.nifi.c2.protocol.api.C2HeartbeatResponse; +import org.apache.nifi.c2.protocol.api.C2OperationAck; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class C2HttpClient implements C2Client { + + private static final Logger logger = LoggerFactory.getLogger(C2HttpClient.class); + private static final MediaType MEDIA_TYPE_APPLICATION_JSON = MediaType.parse("application/json"); + + private final AtomicReference httpClientReference = new AtomicReference<>(); + private final C2ClientConfig clientConfig; + private final C2Serializer serializer; + + public C2HttpClient(C2ClientConfig clientConfig, C2Serializer serializer) { + super(); + this.clientConfig = clientConfig; + this.serializer = serializer; + final OkHttpClient.Builder okHttpClientBuilder = new OkHttpClient.Builder(); + + // Configure request and response logging + HttpLoggingInterceptor logging = new HttpLoggingInterceptor(logger::debug); + logging.setLevel(HttpLoggingInterceptor.Level.BASIC); + okHttpClientBuilder.addInterceptor(logging); + + // Set whether to follow redirects + okHttpClientBuilder.followRedirects(true); + + // Timeouts + okHttpClientBuilder.connectTimeout(clientConfig.getConnectTimeout(), TimeUnit.MILLISECONDS); + okHttpClientBuilder.readTimeout(clientConfig.getReadTimeout(), TimeUnit.MILLISECONDS); + okHttpClientBuilder.callTimeout(clientConfig.getCallTimeout(), TimeUnit.MILLISECONDS); + + // check if the ssl path is set and add the factory if so + if (StringUtils.isNotBlank(clientConfig.getKeystoreFilename())) { + try { + setSslSocketFactory(okHttpClientBuilder); + } catch (Exception e) { + throw new IllegalStateException("OkHttp TLS configuration failed", e); + } + } + + httpClientReference.set(okHttpClientBuilder.build()); + } + + @Override + public Optional publishHeartbeat(C2Heartbeat heartbeat) { + return serializer.serialize(heartbeat).flatMap(this::sendHeartbeat); + } + + private Optional sendHeartbeat(String heartbeat) { + Optional c2HeartbeatResponse = Optional.empty(); + Request request = new Request.Builder() + .post(RequestBody.create(heartbeat, MEDIA_TYPE_APPLICATION_JSON)) + .url(clientConfig.getC2Url()) + .build(); + + try (Response heartbeatResponse = httpClientReference.get().newCall(request).execute()) { + c2HeartbeatResponse = getResponseBody(heartbeatResponse).flatMap(response -> serializer.deserialize(response, C2HeartbeatResponse.class)); + } catch (IOException ce) { + logger.error("Send Heartbeat failed [{}]", clientConfig.getC2Url(), ce); + } + + return c2HeartbeatResponse; + } + + private Optional getResponseBody(Response response) { + String responseBody = null; + + try { + responseBody = response.body().string(); + logger.debug("Received response body {}", responseBody); + } catch (IOException e) { + logger.error("HTTP Request failed", e); + } + + return Optional.ofNullable(responseBody); + } + + private void setSslSocketFactory(OkHttpClient.Builder okHttpClientBuilder) throws Exception { + final String keystoreLocation = clientConfig.getKeystoreFilename(); + final String keystoreType = clientConfig.getKeystoreType(); + final String keystorePass = clientConfig.getKeystorePass(); + + assertKeystorePropertiesSet(keystoreLocation, keystorePass, keystoreType); + + // prepare the keystore + final KeyStore keyStore = KeyStore.getInstance(keystoreType); + + try (FileInputStream keyStoreStream = new FileInputStream(keystoreLocation)) { + keyStore.load(keyStoreStream, keystorePass.toCharArray()); + } + + final KeyManagerFactory keyManagerFactory = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); + keyManagerFactory.init(keyStore, keystorePass.toCharArray()); + + // load truststore + final String truststoreLocation = clientConfig.getTruststoreFilename(); + final String truststorePass = clientConfig.getTruststorePass(); + final String truststoreType = clientConfig.getTruststoreType(); + assertTruststorePropertiesSet(truststoreLocation, truststorePass, truststoreType); + + KeyStore truststore = KeyStore.getInstance(truststoreType); + final TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance("X509"); + truststore.load(new FileInputStream(truststoreLocation), truststorePass.toCharArray()); + trustManagerFactory.init(truststore); + + final X509TrustManager x509TrustManager; + TrustManager[] trustManagers = trustManagerFactory.getTrustManagers(); + if (trustManagers[0] != null) { + x509TrustManager = (X509TrustManager) trustManagers[0]; + } else { + throw new IllegalStateException("List of trust managers is null"); + } + + SSLContext tempSslContext; + try { + tempSslContext = SSLContext.getInstance("TLS"); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException("SSLContext creation failed", e); + } + + final SSLContext sslContext = tempSslContext; + sslContext.init(keyManagerFactory.getKeyManagers(), trustManagerFactory.getTrustManagers(), null); + + final SSLSocketFactory sslSocketFactory = sslContext.getSocketFactory(); + okHttpClientBuilder.sslSocketFactory(sslSocketFactory, x509TrustManager); + } + + private void assertKeystorePropertiesSet(String location, String password, String type) { + if (location == null || location.isEmpty()) { + throw new IllegalArgumentException(clientConfig.getKeystoreFilename() + " is null or is empty"); + } + + if (password == null || password.isEmpty()) { + throw new IllegalArgumentException("The client's keystore filename is set but its password is not (or is empty). If the location is set, the password must also be."); + } + + if (type == null || type.isEmpty()) { + throw new IllegalArgumentException("The client's keystore filename is set but its type is not (or is empty). If the location is set, the type must also be."); + } + } + + private void assertTruststorePropertiesSet(String location, String password, String type) { + if (location == null || location.isEmpty()) { + throw new IllegalArgumentException("The client's truststore filename is not set or is empty"); + } + + if (password == null || password.isEmpty()) { + throw new IllegalArgumentException("The client's truststore filename is set but its password is not (or is empty). If the location is set, the password must also be."); + } + + if (type == null || type.isEmpty()) { + throw new IllegalArgumentException("The client's truststore filename is set but its type is not (or is empty). If the location is set, the type must also be."); + } + } + + @Override + public Optional retrieveUpdateContent(String flowUpdateUrl) { + final Request.Builder requestBuilder = new Request.Builder() + .get() + .url(flowUpdateUrl); + final Request request = requestBuilder.build(); + + ResponseBody body; + try (final Response response = httpClientReference.get().newCall(request).execute()) { + int code = response.code(); + if (code >= 400) { + final String message = String.format("Configuration retrieval failed: HTTP %d %s", code, response.body().string()); + throw new IOException(message); + } + + body = response.body(); + + if (body == null) { + logger.warn("No body returned when pulling a new configuration"); + return Optional.empty(); + } + + return Optional.of(body.bytes()); + } catch (Exception e) { + logger.warn("Configuration retrieval failed", e); + return Optional.empty(); + } + } + + @Override + public void acknowledgeOperation(C2OperationAck operationAck) { + logger.info("Performing acknowledgement request to {} for operation {}", clientConfig.getC2AckUrl(), operationAck.getOperationId()); + serializer.serialize(operationAck) + .map(operationAckBody -> RequestBody.create(operationAckBody, MEDIA_TYPE_APPLICATION_JSON)) + .map(requestBody -> new Request.Builder().post(requestBody).url(clientConfig.getC2AckUrl()).build()) + .ifPresent(this::sendAck); + } + + private void sendAck(Request request) { + try(Response heartbeatResponse = httpClientReference.get().newCall(request).execute()) { + if (!heartbeatResponse.isSuccessful()) { + logger.warn("Acknowledgement was not successful with c2 server [{}] with status code {}", clientConfig.getC2AckUrl(), heartbeatResponse.code()); + } + } catch (IOException e) { + logger.error("Could not transmit ack to c2 server [{}]", clientConfig.getC2AckUrl(), e); + } + } +} diff --git a/c2/c2-client-bundle/c2-client-service/pom.xml b/c2/c2-client-bundle/c2-client-service/pom.xml new file mode 100644 index 0000000000..b45f7d225b --- /dev/null +++ b/c2/c2-client-bundle/c2-client-service/pom.xml @@ -0,0 +1,49 @@ + + + + 4.0.0 + + + c2-client-bundle + org.apache.nifi + 1.17.0-SNAPSHOT + + + c2-client-service + jar + + + + org.apache.nifi + c2-client-api + 1.17.0-SNAPSHOT + provided + + + org.apache.nifi + c2-client-base + 1.17.0-SNAPSHOT + + + org.apache.nifi + c2-client-http + 1.17.0-SNAPSHOT + + + + \ No newline at end of file diff --git a/c2/c2-client-bundle/c2-client-service/src/main/java/org/apache/nifi/c2/client/service/C2ClientService.java b/c2/c2-client-bundle/c2-client-service/src/main/java/org/apache/nifi/c2/client/service/C2ClientService.java new file mode 100644 index 0000000000..a36b7bf3fa --- /dev/null +++ b/c2/c2-client-bundle/c2-client-service/src/main/java/org/apache/nifi/c2/client/service/C2ClientService.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.c2.client.service; + +import java.util.Arrays; +import java.util.List; +import java.util.function.Function; +import org.apache.nifi.c2.client.api.C2Client; +import org.apache.nifi.c2.client.service.model.RuntimeInfoWrapper; +import org.apache.nifi.c2.client.service.operation.C2OperationService; +import org.apache.nifi.c2.client.service.operation.UpdateConfigurationOperationHandler; +import org.apache.nifi.c2.protocol.api.C2Heartbeat; +import org.apache.nifi.c2.protocol.api.C2HeartbeatResponse; +import org.apache.nifi.c2.protocol.api.C2Operation; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class C2ClientService { + + private static final Logger logger = LoggerFactory.getLogger(C2ClientService.class); + + private final C2Client client; + private final C2HeartbeatFactory c2HeartbeatFactory; + private final C2OperationService operationService; + private final UpdateConfigurationOperationHandler updateConfigurationOperationHandler; + + public C2ClientService(C2Client client, C2HeartbeatFactory c2HeartbeatFactory, FlowIdHolder flowIdHolder, Function updateFlow) { + this.client = client; + this.c2HeartbeatFactory = c2HeartbeatFactory; + this.updateConfigurationOperationHandler = new UpdateConfigurationOperationHandler(client, flowIdHolder, updateFlow); + this.operationService = new C2OperationService(Arrays.asList(updateConfigurationOperationHandler)); + } + + public void sendHeartbeat(RuntimeInfoWrapper runtimeInfoWrapper) { + C2Heartbeat c2Heartbeat = c2HeartbeatFactory.create(runtimeInfoWrapper); + client.publishHeartbeat(c2Heartbeat).ifPresent(this::processResponse); + } + + private void processResponse(C2HeartbeatResponse response) { + List requestedOperations = response.getRequestedOperations(); + if (requestedOperations != null && !requestedOperations.isEmpty()) { + logger.info("Received {} operations from the C2 server", requestedOperations.size()); + handleRequestedOperations(requestedOperations); + } else { + logger.trace("No operations received from the C2 server in the server. Nothing to do."); + } + } + + private void handleRequestedOperations(List requestedOperations) { + for (C2Operation requestedOperation : requestedOperations) { + operationService.handleOperation(requestedOperation) + .ifPresent(client::acknowledgeOperation); + } + } +} + diff --git a/c2/c2-client-bundle/c2-client-service/src/main/java/org/apache/nifi/c2/client/service/C2HeartbeatFactory.java b/c2/c2-client-bundle/c2-client-service/src/main/java/org/apache/nifi/c2/client/service/C2HeartbeatFactory.java new file mode 100644 index 0000000000..3fb40813cb --- /dev/null +++ b/c2/c2-client-bundle/c2-client-service/src/main/java/org/apache/nifi/c2/client/service/C2HeartbeatFactory.java @@ -0,0 +1,227 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.c2.client.service; + +import static org.apache.commons.lang3.StringUtils.isNotBlank; + +import java.io.File; +import java.lang.management.ManagementFactory; +import java.lang.management.OperatingSystemMXBean; +import java.net.InetAddress; +import java.net.NetworkInterface; +import java.util.Enumeration; +import java.util.HashSet; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import org.apache.nifi.c2.client.C2ClientConfig; +import org.apache.nifi.c2.client.PersistentUuidGenerator; +import org.apache.nifi.c2.client.service.model.RuntimeInfoWrapper; +import org.apache.nifi.c2.protocol.api.AgentInfo; +import org.apache.nifi.c2.protocol.api.AgentRepositories; +import org.apache.nifi.c2.protocol.api.AgentStatus; +import org.apache.nifi.c2.protocol.api.C2Heartbeat; +import org.apache.nifi.c2.protocol.api.DeviceInfo; +import org.apache.nifi.c2.protocol.api.FlowInfo; +import org.apache.nifi.c2.protocol.api.FlowQueueStatus; +import org.apache.nifi.c2.protocol.api.NetworkInfo; +import org.apache.nifi.c2.protocol.api.SystemInfo; +import org.apache.nifi.c2.protocol.component.api.RuntimeManifest; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class C2HeartbeatFactory { + + private static final Logger logger = LoggerFactory.getLogger(C2HeartbeatFactory.class); + + private static final String AGENT_IDENTIFIER_FILENAME = "agent-identifier"; + private static final String DEVICE_IDENTIFIER_FILENAME = "device-identifier"; + + private final C2ClientConfig clientConfig; + private final FlowIdHolder flowIdHolder; + + private String agentId; + private String deviceId; + private File confDirectory; + + public C2HeartbeatFactory(C2ClientConfig clientConfig, FlowIdHolder flowIdHolder) { + this.clientConfig = clientConfig; + this.flowIdHolder = flowIdHolder; + } + + public C2Heartbeat create(RuntimeInfoWrapper runtimeInfoWrapper) { + C2Heartbeat heartbeat = new C2Heartbeat(); + + heartbeat.setAgentInfo(getAgentInfo(runtimeInfoWrapper.getAgentRepositories(), runtimeInfoWrapper.getManifest())); + heartbeat.setDeviceInfo(generateDeviceInfo()); + heartbeat.setFlowInfo(getFlowInfo(runtimeInfoWrapper.getQueueStatus())); + heartbeat.setCreated(System.currentTimeMillis()); + + return heartbeat; + } + + private FlowInfo getFlowInfo(Map queueStatus) { + FlowInfo flowInfo = new FlowInfo(); + flowInfo.setQueues(queueStatus); + Optional.ofNullable(flowIdHolder.getFlowId()).ifPresent(flowInfo::setFlowId); + return flowInfo; + } + + private AgentInfo getAgentInfo(AgentRepositories repos, RuntimeManifest manifest) { + AgentInfo agentInfo = new AgentInfo(); + agentInfo.setAgentClass(clientConfig.getAgentClass()); + agentInfo.setIdentifier(getAgentId()); + + AgentStatus agentStatus = new AgentStatus(); + agentStatus.setUptime(ManagementFactory.getRuntimeMXBean().getUptime()); + agentStatus.setRepositories(repos); + + agentInfo.setStatus(agentStatus); + agentInfo.setAgentManifest(manifest); + + return agentInfo; + } + + private String getAgentId() { + if (agentId == null) { + String rawAgentId = clientConfig.getAgentIdentifier(); + if (isNotBlank(rawAgentId)) { + agentId = rawAgentId.trim(); + } else { + File idFile = new File(getConfDirectory(), AGENT_IDENTIFIER_FILENAME); + agentId = new PersistentUuidGenerator(idFile).generate(); + } + } + + return agentId; + } + + private DeviceInfo generateDeviceInfo() { + // Populate DeviceInfo + final DeviceInfo deviceInfo = new DeviceInfo(); + deviceInfo.setNetworkInfo(generateNetworkInfo()); + deviceInfo.setIdentifier(getDeviceIdentifier(deviceInfo.getNetworkInfo())); + deviceInfo.setSystemInfo(generateSystemInfo()); + return deviceInfo; + } + + private NetworkInfo generateNetworkInfo() { + NetworkInfo networkInfo = new NetworkInfo(); + try { + // Determine all interfaces + final Enumeration networkInterfaces = NetworkInterface.getNetworkInterfaces(); + + final Set operationIfaces = new HashSet<>(); + + // Determine eligible interfaces + while (networkInterfaces.hasMoreElements()) { + final NetworkInterface networkInterface = networkInterfaces.nextElement(); + if (!networkInterface.isLoopback() && networkInterface.isUp()) { + operationIfaces.add(networkInterface); + } + } + logger.trace("Have {} interfaces with names {}", operationIfaces.size(), + operationIfaces.stream() + .map(NetworkInterface::getName) + .collect(Collectors.toSet()) + ); + + if (!operationIfaces.isEmpty()) { + if (operationIfaces.size() > 1) { + logger.debug("Instance has multiple interfaces. Generated information may be non-deterministic."); + } + + NetworkInterface iface = operationIfaces.iterator().next(); + Enumeration inetAddresses = iface.getInetAddresses(); + while (inetAddresses.hasMoreElements()) { + InetAddress inetAddress = inetAddresses.nextElement(); + String hostAddress = inetAddress.getHostAddress(); + String hostName = inetAddress.getHostName(); + byte[] address = inetAddress.getAddress(); + String canonicalHostName = inetAddress.getCanonicalHostName(); + + networkInfo.setDeviceId(iface.getName()); + networkInfo.setHostname(hostName); + networkInfo.setIpAddress(hostAddress); + } + } + } catch ( + Exception e) { + logger.error("Network Interface processing failed", e); + } + return networkInfo; + } + + private String getDeviceIdentifier(NetworkInfo networkInfo) { + if (deviceId == null) { + if (networkInfo.getDeviceId() != null) { + try { + final NetworkInterface netInterface = NetworkInterface.getByName(networkInfo.getDeviceId()); + byte[] hardwareAddress = netInterface.getHardwareAddress(); + final StringBuilder macBuilder = new StringBuilder(); + if (hardwareAddress != null) { + for (byte address : hardwareAddress) { + macBuilder.append(String.format("%02X", address)); + } + } + deviceId = macBuilder.toString(); + } catch (Exception e) { + logger.warn("Could not determine device identifier. Generating a unique ID", e); + deviceId = getConfiguredDeviceId(); + } + } else { + deviceId = getConfiguredDeviceId(); + } + } + + return deviceId; + } + + private String getConfiguredDeviceId() { + File idFile = new File(getConfDirectory(), DEVICE_IDENTIFIER_FILENAME); + return new PersistentUuidGenerator(idFile).generate(); + } + + private SystemInfo generateSystemInfo() { + SystemInfo systemInfo = new SystemInfo(); + systemInfo.setPhysicalMem(Runtime.getRuntime().maxMemory()); + systemInfo.setMemoryUsage(Runtime.getRuntime().maxMemory() - Runtime.getRuntime().freeMemory()); + systemInfo.setvCores(Runtime.getRuntime().availableProcessors()); + + OperatingSystemMXBean osMXBean = ManagementFactory.getOperatingSystemMXBean(); + systemInfo.setMachineArch(osMXBean.getArch()); + systemInfo.setOperatingSystem(osMXBean.getName()); + systemInfo.setCpuUtilization(osMXBean.getSystemLoadAverage() / (double) osMXBean.getAvailableProcessors()); + + return systemInfo; + } + + private File getConfDirectory() { + if (confDirectory == null) { + String configDirectoryName = clientConfig.getConfDirectory(); + File configDirectory = new File(configDirectoryName); + if (!configDirectory.exists() || !configDirectory.isDirectory()) { + throw new IllegalStateException("Specified conf directory " + configDirectoryName + " does not exist or is not a directory."); + } + + confDirectory = configDirectory; + } + + return confDirectory; + } +} diff --git a/c2/c2-client-bundle/c2-client-service/src/main/java/org/apache/nifi/c2/client/service/FlowIdHolder.java b/c2/c2-client-bundle/c2-client-service/src/main/java/org/apache/nifi/c2/client/service/FlowIdHolder.java new file mode 100644 index 0000000000..30d194f720 --- /dev/null +++ b/c2/c2-client-bundle/c2-client-service/src/main/java/org/apache/nifi/c2/client/service/FlowIdHolder.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.c2.client.service; + +import static java.util.Collections.singletonList; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.List; +import org.apache.nifi.c2.util.FileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class FlowIdHolder { + private static final Logger LOGGER = LoggerFactory.getLogger(FlowIdHolder.class); + private static final String FLOW_IDENTIFIER_FILENAME = "flow-identifier"; + + private volatile String flowId; + private final String configDirectoryName; + + public FlowIdHolder(String configDirectoryName) { + this.configDirectoryName = configDirectoryName; + this.flowId = readFlowId(); + } + + public String getFlowId() { + return flowId; + } + + public void setFlowId(String flowId) { + this.flowId = flowId; + persistFlowId(flowId); + } + + private void persistFlowId(String flowId) { + File flowIdFile = new File(configDirectoryName, FLOW_IDENTIFIER_FILENAME); + try { + FileUtils.ensureDirectoryExistAndCanAccess(flowIdFile.getParentFile()); + saveFlowId(flowIdFile, flowId); + } catch (IOException e) { + LOGGER.error("Persisting Flow [{}] failed", flowId, e); + } + } + + private void saveFlowId(File flowUpdateInfoFile, String flowId) { + try { + Files.write(flowUpdateInfoFile.toPath(), singletonList(flowId)); + } catch (IOException e) { + LOGGER.error("Writing Flow [{}] failed", flowId, e); + } + } + + private String readFlowId() { + File flowUpdateInfoFile = new File(configDirectoryName, FLOW_IDENTIFIER_FILENAME); + String flowId = null; + if (flowUpdateInfoFile.exists()) { + try { + List fileLines = Files.readAllLines(flowUpdateInfoFile.toPath()); + if (fileLines.size() != 1) { + throw new IllegalStateException(String.format("The file %s for the persisted flow id has the incorrect format.", flowUpdateInfoFile)); + } + flowId = fileLines.get(0); + } catch (IOException e) { + throw new IllegalStateException(String.format("Could not read file %s for persisted flow id.", flowUpdateInfoFile), e); + } + } + return flowId; + } +} diff --git a/c2/c2-client-bundle/c2-client-service/src/main/java/org/apache/nifi/c2/client/service/model/RuntimeInfoWrapper.java b/c2/c2-client-bundle/c2-client-service/src/main/java/org/apache/nifi/c2/client/service/model/RuntimeInfoWrapper.java new file mode 100644 index 0000000000..c017ac0d11 --- /dev/null +++ b/c2/c2-client-bundle/c2-client-service/src/main/java/org/apache/nifi/c2/client/service/model/RuntimeInfoWrapper.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.c2.client.service.model; + +import java.util.Map; +import org.apache.nifi.c2.protocol.api.AgentRepositories; +import org.apache.nifi.c2.protocol.api.FlowQueueStatus; +import org.apache.nifi.c2.protocol.component.api.RuntimeManifest; + +public class RuntimeInfoWrapper { + final AgentRepositories repos; + final RuntimeManifest manifest; + final Map queueStatus; + + public RuntimeInfoWrapper(AgentRepositories repos, RuntimeManifest manifest, Map queueStatus) { + this.repos = repos; + this.manifest = manifest; + this.queueStatus = queueStatus; + } + + public AgentRepositories getAgentRepositories() { + return repos; + } + + public RuntimeManifest getManifest() { + return manifest; + } + + public Map getQueueStatus() { + return queueStatus; + } +} diff --git a/c2/c2-client-bundle/c2-client-service/src/main/java/org/apache/nifi/c2/client/service/operation/C2OperationHandler.java b/c2/c2-client-bundle/c2-client-service/src/main/java/org/apache/nifi/c2/client/service/operation/C2OperationHandler.java new file mode 100644 index 0000000000..e9f2db29fb --- /dev/null +++ b/c2/c2-client-bundle/c2-client-service/src/main/java/org/apache/nifi/c2/client/service/operation/C2OperationHandler.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.c2.client.service.operation; + +import org.apache.nifi.c2.protocol.api.C2Operation; +import org.apache.nifi.c2.protocol.api.C2OperationAck; +import org.apache.nifi.c2.protocol.api.OperandType; +import org.apache.nifi.c2.protocol.api.OperationType; + +/** + * Handler interface for the different operation types + */ +public interface C2OperationHandler { + + /** + * Returns the supported OperationType by the handler + * + * @return the type of the operation + */ + OperationType getOperationType(); + + /** + * Returns the supported OperandType by the handler + * + * @return the type of the operand + */ + OperandType getOperandType(); + + /** + * Handler logic for the specific C2Operation + * + * @param operation the C2Operation to be handled + * @return the result of the operation handling + */ + C2OperationAck handle(C2Operation operation); +} diff --git a/c2/c2-client-bundle/c2-client-service/src/main/java/org/apache/nifi/c2/client/service/operation/C2OperationService.java b/c2/c2-client-bundle/c2-client-service/src/main/java/org/apache/nifi/c2/client/service/operation/C2OperationService.java new file mode 100644 index 0000000000..fadc8ad79b --- /dev/null +++ b/c2/c2-client-bundle/c2-client-service/src/main/java/org/apache/nifi/c2/client/service/operation/C2OperationService.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.c2.client.service.operation; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import org.apache.nifi.c2.protocol.api.C2Operation; +import org.apache.nifi.c2.protocol.api.C2OperationAck; +import org.apache.nifi.c2.protocol.api.OperandType; +import org.apache.nifi.c2.protocol.api.OperationType; + +public class C2OperationService { + + private final Map> handlerMap = new HashMap<>(); + + public C2OperationService(List handlers) { + for (C2OperationHandler handler : handlers) { + handlerMap.computeIfAbsent(handler.getOperationType(), x -> new HashMap<>()).put(handler.getOperandType(), handler); + } + } + + public Optional handleOperation(C2Operation operation) { + return getHandlerForOperation(operation) + .map(handler -> handler.handle(operation)); + } + + private Optional getHandlerForOperation(C2Operation operation) { + return Optional.ofNullable(handlerMap.get(operation.getOperation())) + .map(operandMap -> operandMap.get(operation.getOperand())); + } +} diff --git a/c2/c2-client-bundle/c2-client-service/src/main/java/org/apache/nifi/c2/client/service/operation/UpdateConfigurationOperationHandler.java b/c2/c2-client-bundle/c2-client-service/src/main/java/org/apache/nifi/c2/client/service/operation/UpdateConfigurationOperationHandler.java new file mode 100644 index 0000000000..b58e07547e --- /dev/null +++ b/c2/c2-client-bundle/c2-client-service/src/main/java/org/apache/nifi/c2/client/service/operation/UpdateConfigurationOperationHandler.java @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.c2.client.service.operation; + +import static org.apache.commons.lang3.StringUtils.EMPTY; +import static org.apache.nifi.c2.protocol.api.OperandType.CONFIGURATION; +import static org.apache.nifi.c2.protocol.api.OperationType.UPDATE; + +import java.net.URI; +import java.util.Optional; +import java.util.function.Function; +import org.apache.nifi.c2.client.api.C2Client; +import org.apache.nifi.c2.client.service.FlowIdHolder; +import org.apache.nifi.c2.protocol.api.C2Operation; +import org.apache.nifi.c2.protocol.api.C2OperationAck; +import org.apache.nifi.c2.protocol.api.C2OperationState; +import org.apache.nifi.c2.protocol.api.OperandType; +import org.apache.nifi.c2.protocol.api.OperationType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class UpdateConfigurationOperationHandler implements C2OperationHandler { + + private static final Logger logger = LoggerFactory.getLogger(UpdateConfigurationOperationHandler.class); + + private static final String LOCATION = "location"; + + private final C2Client client; + private final Function updateFlow; + private final FlowIdHolder flowIdHolder; + + public UpdateConfigurationOperationHandler(C2Client client, FlowIdHolder flowIdHolder, Function updateFlow) { + this.client = client; + this.updateFlow = updateFlow; + this.flowIdHolder = flowIdHolder; + } + + @Override + public OperationType getOperationType() { + return UPDATE; + } + + @Override + public OperandType getOperandType() { + return CONFIGURATION; + } + + @Override + public C2OperationAck handle(C2Operation operation) { + String opIdentifier = Optional.ofNullable(operation.getIdentifier()) + .orElse(EMPTY); + C2OperationAck operationAck = new C2OperationAck(); + C2OperationState state = new C2OperationState(); + operationAck.setOperationState(state); + operationAck.setOperationId(opIdentifier); + + String updateLocation = Optional.ofNullable(operation.getArgs()) + .map(map -> map.get(LOCATION)) + .orElse(EMPTY); + + String newFlowId = parseFlowId(updateLocation); + if (flowIdHolder.getFlowId() == null || !flowIdHolder.getFlowId().equals(newFlowId)) { + logger.info("Will perform flow update from {} for operation #{}. Previous flow id was {}, replacing with new id {}", updateLocation, opIdentifier, + flowIdHolder.getFlowId() == null ? "not set" : flowIdHolder.getFlowId(), newFlowId); + } else { + logger.info("Flow is current, no update is necessary..."); + } + + flowIdHolder.setFlowId(newFlowId); + Optional updateContent = client.retrieveUpdateContent(updateLocation); + if (updateContent.isPresent()) { + if (updateFlow.apply(updateContent.get())) { + state.setState(C2OperationState.OperationState.FULLY_APPLIED); + logger.debug("Update configuration applied for operation #{}.", opIdentifier); + } else { + state.setState(C2OperationState.OperationState.NOT_APPLIED); + logger.error("Update resulted in error for operation #{}.", opIdentifier); + } + } else { + state.setState(C2OperationState.OperationState.NOT_APPLIED); + logger.error("Update content retrieval resulted in empty content so flow update was omitted for operation #{}.", opIdentifier); + } + + return operationAck; + } + + private String parseFlowId(String flowUpdateUrl) { + try { + URI flowUri = new URI(flowUpdateUrl); + String flowUriPath = flowUri.getPath(); + String[] split = flowUriPath.split("/"); + if (split.length > 4) { + return split[4]; + } else { + throw new IllegalArgumentException(String.format("Flow Update URL format unexpected [%s]", flowUpdateUrl)); + } + } catch (Exception e) { + throw new IllegalStateException("Could not get flow id from the provided URL", e); + } + } +} diff --git a/c2/c2-client-bundle/pom.xml b/c2/c2-client-bundle/pom.xml new file mode 100644 index 0000000000..4781e5d4f4 --- /dev/null +++ b/c2/c2-client-bundle/pom.xml @@ -0,0 +1,36 @@ + + + + 4.0.0 + + + c2 + org.apache.nifi + 1.17.0-SNAPSHOT + + + c2-client-bundle + pom + + + c2-client-api + c2-client-base + c2-client-http + c2-client-service + + diff --git a/c2/c2-protocol/c2-protocol-api/pom.xml b/c2/c2-protocol/c2-protocol-api/pom.xml index 5fb3fb7ccd..b48d5c93c8 100644 --- a/c2/c2-protocol/c2-protocol-api/pom.xml +++ b/c2/c2-protocol/c2-protocol-api/pom.xml @@ -33,5 +33,9 @@ limitations under the License. c2-protocol-component-api 1.17.0-SNAPSHOT + + org.apache.commons + commons-lang3 + diff --git a/c2/c2-protocol/c2-protocol-api/src/main/java/org/apache/nifi/c2/protocol/api/OperandType.java b/c2/c2-protocol/c2-protocol-api/src/main/java/org/apache/nifi/c2/protocol/api/OperandType.java index bac9403d76..389d3bed5a 100644 --- a/c2/c2-protocol/c2-protocol-api/src/main/java/org/apache/nifi/c2/protocol/api/OperandType.java +++ b/c2/c2-protocol/c2-protocol-api/src/main/java/org/apache/nifi/c2/protocol/api/OperandType.java @@ -33,5 +33,10 @@ public enum OperandType { .filter(operandType -> operandType.name().equalsIgnoreCase(value)) .findAny(); } + + @Override + public String toString() { + return super.toString().toLowerCase(); + } } diff --git a/c2/pom.xml b/c2/pom.xml index 37feecdaa4..712d69e6db 100644 --- a/c2/pom.xml +++ b/c2/pom.xml @@ -29,6 +29,7 @@ limitations under the License. c2-protocol + c2-client-bundle diff --git a/minifi/minifi-assembly/pom.xml b/minifi/minifi-assembly/pom.xml index 4e0f2965e2..6109eb8210 100644 --- a/minifi/minifi-assembly/pom.xml +++ b/minifi/minifi-assembly/pom.xml @@ -122,6 +122,10 @@ limitations under the License. org.apache.nifi.minifi minifi-framework-api + + org.apache.nifi + c2-client-api + org.apache.nifi.minifi minifi-framework-nar @@ -210,6 +214,14 @@ limitations under the License. commons-io commons-io + + com.fasterxml.jackson.core + jackson-core + + + com.fasterxml.jackson.core + jackson-databind + diff --git a/minifi/minifi-bootstrap/pom.xml b/minifi/minifi-bootstrap/pom.xml index 693d128072..315fa89373 100644 --- a/minifi/minifi-bootstrap/pom.xml +++ b/minifi/minifi-bootstrap/pom.xml @@ -40,6 +40,11 @@ limitations under the License. nifi-api compile + + org.apache.nifi + c2-client-api + compile + org.apache.nifi nifi-bootstrap-utils @@ -95,6 +100,15 @@ limitations under the License. commons-io provided + + + com.fasterxml.jackson.core + jackson-core + + + com.fasterxml.jackson.core + jackson-databind + diff --git a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/BootstrapCodec.java b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/BootstrapCodec.java deleted file mode 100644 index 6e8ae91948..0000000000 --- a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/BootstrapCodec.java +++ /dev/null @@ -1,122 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.minifi.bootstrap; - -import org.apache.nifi.minifi.bootstrap.exception.InvalidCommandException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.BufferedReader; -import java.io.BufferedWriter; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.OutputStream; -import java.io.OutputStreamWriter; -import java.util.Arrays; - -public class BootstrapCodec { - - private final RunMiNiFi runner; - private final BufferedReader reader; - private final BufferedWriter writer; - private final Logger logger = LoggerFactory.getLogger(BootstrapCodec.class); - - public BootstrapCodec(final RunMiNiFi runner, final InputStream in, final OutputStream out) { - this.runner = runner; - this.reader = new BufferedReader(new InputStreamReader(in)); - this.writer = new BufferedWriter(new OutputStreamWriter(out)); - } - - public void communicate() throws IOException { - final String line = reader.readLine(); - final String[] splits = line.split(" "); - if (splits.length < 0) { - throw new IOException("Received invalid command from MiNiFi: " + line); - } - - final String cmd = splits[0]; - final String[] args; - if (splits.length == 1) { - args = new String[0]; - } else { - args = Arrays.copyOfRange(splits, 1, splits.length); - } - - try { - processRequest(cmd, args); - } catch (final InvalidCommandException ice) { - throw new IOException("Received invalid command from MiNiFi: " + line + " : " + (ice.getMessage() == null ? "" : "Details: " + ice.toString())); - } - } - - private void processRequest(final String cmd, final String[] args) throws InvalidCommandException, IOException { - switch (cmd) { - case "PORT": { - logger.debug("Received 'PORT' command from MINIFI"); - if (args.length != 2) { - throw new InvalidCommandException(); - } - - final int port; - try { - port = Integer.parseInt(args[0]); - } catch (final NumberFormatException nfe) { - throw new InvalidCommandException("Invalid Port number; should be integer between 1 and 65535"); - } - - if (port < 1 || port > 65535) { - throw new InvalidCommandException("Invalid Port number; should be integer between 1 and 65535"); - } - - final String secretKey = args[1]; - - runner.setMiNiFiCommandControlPort(port, secretKey); - writer.write("OK"); - writer.newLine(); - writer.flush(); - } - break; - case "STARTED": { - logger.debug("Received 'STARTED' command from MINIFI"); - if (args.length != 1) { - throw new InvalidCommandException("STARTED command must contain a status argument"); - } - - if (!"true".equals(args[0]) && !"false".equals(args[0])) { - throw new InvalidCommandException("Invalid status for STARTED command; should be true or false, but was '" + args[0] + "'"); - } - - final boolean started = Boolean.parseBoolean(args[0]); - runner.setNiFiStarted(started); - writer.write("OK"); - writer.newLine(); - writer.flush(); - } - break; - case "SHUTDOWN": { - logger.debug("Received 'SHUTDOWN' command from MINIFI"); - runner.shutdownChangeNotifier(); - runner.shutdownPeriodicStatusReporters(); - writer.write("OK"); - writer.newLine(); - writer.flush(); - } - break; - } - } -} diff --git a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/BootstrapCommand.java b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/BootstrapCommand.java new file mode 100644 index 0000000000..43bb5bd870 --- /dev/null +++ b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/BootstrapCommand.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.minifi.bootstrap; + +import java.util.Optional; + +public enum BootstrapCommand { + START, RUN, STOP, STATUS, DUMP, RESTART, ENV, FLOWSTATUS, UNKNOWN; + + public static Optional fromString(String val) { + return Optional.ofNullable(val).map(String::toUpperCase).map(BootstrapCommand::valueOf).filter(command -> command != UNKNOWN); + } +} diff --git a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/MiNiFiListener.java b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/MiNiFiListener.java deleted file mode 100644 index ea760bec7b..0000000000 --- a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/MiNiFiListener.java +++ /dev/null @@ -1,141 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.minifi.bootstrap; - -import org.apache.nifi.minifi.bootstrap.util.LimitingInputStream; - -import java.io.IOException; -import java.io.InputStream; -import java.net.InetSocketAddress; -import java.net.ServerSocket; -import java.net.Socket; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.TimeUnit; - -public class MiNiFiListener { - - private ServerSocket serverSocket; - private volatile Listener listener; - - int start(final RunMiNiFi runner) throws IOException { - serverSocket = new ServerSocket(); - serverSocket.bind(new InetSocketAddress("localhost", 0)); - - final int localPort = serverSocket.getLocalPort(); - listener = new Listener(serverSocket, runner); - final Thread listenThread = new Thread(listener); - listenThread.setName("Listen to MiNiFi"); - listenThread.setDaemon(true); - listenThread.start(); - return localPort; - } - - public void stop() throws IOException { - final Listener listener = this.listener; - if (listener == null) { - return; - } - - listener.stop(); - } - - private class Listener implements Runnable { - - private final ServerSocket serverSocket; - private final ExecutorService executor; - private final RunMiNiFi runner; - private volatile boolean stopped = false; - - public Listener(final ServerSocket serverSocket, final RunMiNiFi runner) { - this.serverSocket = serverSocket; - this.executor = Executors.newFixedThreadPool(2, new ThreadFactory() { - @Override - public Thread newThread(final Runnable runnable) { - final Thread t = Executors.defaultThreadFactory().newThread(runnable); - t.setDaemon(true); - t.setName("MiNiFi Bootstrap Command Listener"); - return t; - } - }); - - this.runner = runner; - } - - public void stop() throws IOException { - stopped = true; - - executor.shutdown(); - try { - executor.awaitTermination(3, TimeUnit.SECONDS); - } catch (final InterruptedException ie) { - } - - serverSocket.close(); - } - - @Override - public void run() { - while (!serverSocket.isClosed()) { - try { - if (stopped) { - return; - } - - final Socket socket; - try { - socket = serverSocket.accept(); - } catch (final IOException ioe) { - if (stopped) { - return; - } - - throw ioe; - } - - executor.submit(new Runnable() { - @Override - public void run() { - try { - // we want to ensure that we don't try to read data from an InputStream directly - // by a BufferedReader because any user on the system could open a socket and send - // a multi-gigabyte file without any new lines in order to crash the Bootstrap, - // which in turn may cause the Shutdown Hook to shutdown MiNiFi. - // So we will limit the amount of data to read to 4 KB - final InputStream limitingIn = new LimitingInputStream(socket.getInputStream(), 4096); - final BootstrapCodec codec = new BootstrapCodec(runner, limitingIn, socket.getOutputStream()); - codec.communicate(); - } catch (final Throwable t) { - System.out.println("Failed to communicate with MiNiFi due to " + t); - t.printStackTrace(); - } finally { - try { - socket.close(); - } catch (final IOException ioe) { - } - } - } - }); - } catch (final Throwable t) { - System.err.println("Failed to receive information from MiNiFi due to " + t); - t.printStackTrace(); - } - } - } - } -} diff --git a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/MiNiFiParameters.java b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/MiNiFiParameters.java new file mode 100644 index 0000000000..9fa7bf666a --- /dev/null +++ b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/MiNiFiParameters.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.minifi.bootstrap; + +import java.util.Objects; + +public class MiNiFiParameters { + + private volatile int ccPort; + private volatile long minifiPid; + private volatile String secretKey; + + public MiNiFiParameters(int ccPort, long minifiPid, String secretKey) { + this.ccPort = ccPort; + this.minifiPid = minifiPid; + this.secretKey = secretKey; + } + + public int getMiNiFiPort() { + return ccPort; + } + + public void setMiNiFiPort(int ccPort) { + this.ccPort = ccPort; + } + + public long getMinifiPid() { + return minifiPid; + } + + public void setMinifiPid(long minifiPid) { + this.minifiPid = minifiPid; + } + + public String getSecretKey() { + return secretKey; + } + + public void setSecretKey(String secretKey) { + this.secretKey = secretKey; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + MiNiFiParameters that = (MiNiFiParameters) o; + return ccPort == that.ccPort && minifiPid == that.minifiPid && Objects.equals(secretKey, that.secretKey); + } + + @Override + public int hashCode() { + return Objects.hash(ccPort, minifiPid, secretKey); + } + + @Override + public String toString() { + return "MiNiFiParameters{" + + "ccPort=" + ccPort + + ", minifiPid=" + minifiPid + + '}'; + } +} diff --git a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/MiNiFiStatus.java b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/MiNiFiStatus.java new file mode 100644 index 0000000000..29b372dca4 --- /dev/null +++ b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/MiNiFiStatus.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.minifi.bootstrap; + +public class MiNiFiStatus { + + private final Integer port; + private final Long pid; + private final boolean respondingToPing; + private final boolean processRunning; + + public MiNiFiStatus() { + this.port = null; + this.pid = null; + this.respondingToPing = false; + this.processRunning = false; + } + + public MiNiFiStatus(Integer port, Long pid, boolean respondingToPing, boolean processRunning) { + this.port = port; + this.pid = pid; + this.respondingToPing = respondingToPing; + this.processRunning = processRunning; + } + + public Long getPid() { + return pid; + } + + public Integer getPort() { + return port; + } + + public boolean isRespondingToPing() { + return respondingToPing; + } + + public boolean isProcessRunning() { + return processRunning; + } +} diff --git a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/RunMiNiFi.java b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/RunMiNiFi.java index e9931d9f9e..061a0a7e36 100644 --- a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/RunMiNiFi.java +++ b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/RunMiNiFi.java @@ -16,70 +16,35 @@ */ package org.apache.nifi.minifi.bootstrap; -import org.apache.commons.io.input.TeeInputStream; -import org.apache.commons.lang3.StringUtils; -import org.apache.nifi.bootstrap.util.OSUtils; -import org.apache.nifi.minifi.bootstrap.configuration.ConfigurationChangeCoordinator; -import org.apache.nifi.minifi.bootstrap.configuration.ConfigurationChangeException; -import org.apache.nifi.minifi.bootstrap.configuration.ConfigurationChangeListener; -import org.apache.nifi.minifi.bootstrap.status.PeriodicStatusReporter; -import org.apache.nifi.minifi.bootstrap.util.ConfigTransformer; -import org.apache.nifi.minifi.commons.status.FlowStatusReport; -import org.apache.nifi.util.Tuple; -import org.apache.nifi.util.file.FileUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import static java.util.Collections.singleton; -import java.io.BufferedReader; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.EOFException; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.FilenameFilter; import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.ObjectInputStream; -import java.io.OutputStream; -import java.io.Reader; -import java.lang.reflect.Method; -import java.net.InetAddress; -import java.net.InetSocketAddress; -import java.net.Socket; -import java.net.SocketTimeoutException; import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.nio.file.attribute.PosixFilePermission; -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; +import java.util.Optional; import java.util.Properties; -import java.util.Set; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; -import java.util.concurrent.locks.Condition; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; - -import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; -import static org.apache.nifi.minifi.commons.schema.common.BootstrapPropertyKeys.STATUS_REPORTER_COMPONENTS_KEY; +import org.apache.nifi.c2.client.api.ConfigurationFileHolder; +import org.apache.nifi.minifi.bootstrap.command.CommandRunnerFactory; +import org.apache.nifi.minifi.bootstrap.configuration.ConfigurationChangeCoordinator; +import org.apache.nifi.minifi.bootstrap.service.BootstrapFileProvider; +import org.apache.nifi.minifi.bootstrap.service.CurrentPortProvider; +import org.apache.nifi.minifi.bootstrap.service.GracefulShutdownParameterProvider; +import org.apache.nifi.minifi.bootstrap.service.MiNiFiCommandSender; +import org.apache.nifi.minifi.bootstrap.service.MiNiFiConfigurationChangeListener; +import org.apache.nifi.minifi.bootstrap.service.MiNiFiExecCommandProvider; +import org.apache.nifi.minifi.bootstrap.service.MiNiFiStatusProvider; +import org.apache.nifi.minifi.bootstrap.service.MiNiFiStdLogHandler; +import org.apache.nifi.minifi.bootstrap.service.PeriodicStatusReporterManager; +import org.apache.nifi.minifi.bootstrap.service.ReloadService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** *

@@ -96,91 +61,82 @@ import static org.apache.nifi.minifi.commons.schema.common.BootstrapPropertyKeys *

* If the {@code bootstrap.conf} file cannot be found, throws a {@code FileNotFoundException}. */ -public class RunMiNiFi implements QueryableStatusAggregator, ConfigurationFileHolder { - - public static final String DEFAULT_CONFIG_FILE = "./conf/bootstrap.conf"; - public static final String DEFAULT_NIFI_PROPS_FILE = "./conf/nifi.properties"; - public static final String DEFAULT_JAVA_CMD = "java"; - public static final String DEFAULT_PID_DIR = "bin"; - public static final String DEFAULT_LOG_DIR = "./logs"; - +public class RunMiNiFi implements ConfigurationFileHolder { + // used for logging initial info; these will be logged to console by default when the app is started + public static final Logger CMD_LOGGER = LoggerFactory.getLogger("org.apache.nifi.minifi.bootstrap.Command"); + // used for logging all info. These by default will be written to the log file + public static final Logger DEFAULT_LOGGER = LoggerFactory.getLogger(RunMiNiFi.class); public static final String CONF_DIR_KEY = "conf.dir"; - public static final String MINIFI_CONFIG_FILE_KEY = "nifi.minifi.config"; + public static final String STATUS_FILE_PID_KEY = "pid"; + public static final int UNINITIALIZED = -1; + private static final String STATUS_FILE_PORT_KEY = "port"; + private static final String STATUS_FILE_SECRET_KEY = "secret.key"; - public static final String GRACEFUL_SHUTDOWN_PROP = "graceful.shutdown.seconds"; - public static final String DEFAULT_GRACEFUL_SHUTDOWN_VALUE = "20"; - - public static final String MINIFI_PID_DIR_PROP = "org.apache.nifi.minifi.bootstrap.config.pid.dir"; - - public static final String MINIFI_PID_FILE_NAME = "minifi.pid"; - public static final String MINIFI_STATUS_FILE_NAME = "minifi.status"; - public static final String MINIFI_LOCK_FILE_NAME = "minifi.lock"; - - public static final String PID_KEY = "pid"; - - public static final int STARTUP_WAIT_SECONDS = 60; - - public static final String SHUTDOWN_CMD = "SHUTDOWN"; - public static final String RELOAD_CMD = "RELOAD"; - public static final String PING_CMD = "PING"; - public static final String DUMP_CMD = "DUMP"; - public static final String FLOW_STATUS_REPORT_CMD = "FLOW_STATUS_REPORT"; - - private static final int UNINITIALIZED_CC_PORT = -1; - - private volatile boolean autoRestartNiFi = true; - private volatile int ccPort = UNINITIALIZED_CC_PORT; - private volatile long minifiPid = -1L; - private volatile String secretKey; - private volatile ShutdownHook shutdownHook; - private volatile boolean nifiStarted; - - private final Lock startedLock = new ReentrantLock(); - private final Lock lock = new ReentrantLock(); - private final Condition startupCondition = lock.newCondition(); - private final File bootstrapConfigFile; - - // used for logging initial info; these will be logged to console by default when the app is started - private final Logger cmdLogger = LoggerFactory.getLogger("org.apache.nifi.minifi.bootstrap.Command"); - // used for logging all info. These by default will be written to the log file - private final Logger defaultLogger = LoggerFactory.getLogger(RunMiNiFi.class); - - - private final ExecutorService loggingExecutor; - private volatile Set> loggingFutures = new HashSet<>(2); - private volatile int gracefulShutdownSeconds; - - private Set periodicStatusReporters; - - private ConfigurationChangeCoordinator changeCoordinator; - private MiNiFiConfigurationChangeListener changeListener; - + private final BootstrapFileProvider bootstrapFileProvider; + private final ConfigurationChangeCoordinator configurationChangeCoordinator; + private final CommandRunnerFactory commandRunnerFactory; private final AtomicReference currentConfigFileReference = new AtomicReference<>(); + private final MiNiFiParameters miNiFiParameters; + private final PeriodicStatusReporterManager periodicStatusReporterManager; + private final ReloadService reloadService; + private volatile Boolean autoRestartNiFi = true; + private volatile boolean nifiStarted; + private final Lock startedLock = new ReentrantLock(); + // Is set to true after the MiNiFi instance shuts down in preparation to be reloaded. Will be set to false after MiNiFi is successfully started again. + private final AtomicBoolean reloading = new AtomicBoolean(false); - @Override - public AtomicReference getConfigFileReference() { - return currentConfigFileReference; + public RunMiNiFi(File bootstrapConfigFile) throws IOException { + bootstrapFileProvider = new BootstrapFileProvider(bootstrapConfigFile); + + Properties properties = bootstrapFileProvider.getStatusProperties(); + + miNiFiParameters = new MiNiFiParameters( + Optional.ofNullable(properties.getProperty(STATUS_FILE_PORT_KEY)).map(Integer::parseInt).orElse(UNINITIALIZED), + Optional.ofNullable(properties.getProperty(STATUS_FILE_PID_KEY)).map(Integer::parseInt).orElse(UNINITIALIZED), + properties.getProperty(STATUS_FILE_SECRET_KEY) + ); + + MiNiFiCommandSender miNiFiCommandSender = new MiNiFiCommandSender(miNiFiParameters, getObjectMapper()); + MiNiFiStatusProvider miNiFiStatusProvider = new MiNiFiStatusProvider(miNiFiCommandSender); + periodicStatusReporterManager = + new PeriodicStatusReporterManager(bootstrapFileProvider.getBootstrapProperties(), miNiFiStatusProvider, miNiFiCommandSender, miNiFiParameters); + configurationChangeCoordinator = new ConfigurationChangeCoordinator(bootstrapFileProvider.getBootstrapProperties(), this, + singleton(new MiNiFiConfigurationChangeListener(this, DEFAULT_LOGGER, bootstrapFileProvider))); + + CurrentPortProvider currentPortProvider = new CurrentPortProvider(miNiFiCommandSender, miNiFiParameters); + GracefulShutdownParameterProvider gracefulShutdownParameterProvider = new GracefulShutdownParameterProvider(bootstrapFileProvider); + reloadService = new ReloadService(bootstrapFileProvider, miNiFiParameters, miNiFiCommandSender, currentPortProvider, gracefulShutdownParameterProvider, this); + commandRunnerFactory = new CommandRunnerFactory(miNiFiCommandSender, currentPortProvider, miNiFiParameters, miNiFiStatusProvider, periodicStatusReporterManager, + bootstrapFileProvider, new MiNiFiStdLogHandler(), bootstrapConfigFile, this, gracefulShutdownParameterProvider, + new MiNiFiExecCommandProvider(bootstrapFileProvider)); } - // Is set to true after the MiNiFi instance shuts down in preparation to be reloaded. Will be set to false after MiNiFi is successfully started again. - private AtomicBoolean reloading = new AtomicBoolean(false); + public int run(BootstrapCommand command, String... args) { + return commandRunnerFactory.getRunner(command).runCommand(args); + } - private final SimpleDateFormat sdf = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss.SSS"); + public static void main(String[] args) { + if (args.length < 1 || args.length > 3) { + printUsage(); + return; + } - public RunMiNiFi(final File bootstrapConfigFile) throws IOException { - this.bootstrapConfigFile = bootstrapConfigFile; + Optional cmd = BootstrapCommand.fromString(args[0]); + if (!cmd.isPresent()) { + printUsage(); + return; + } - loggingExecutor = Executors.newFixedThreadPool(2, new ThreadFactory() { - @Override - public Thread newThread(final Runnable runnable) { - final Thread t = Executors.defaultThreadFactory().newThread(runnable); - t.setDaemon(true); - t.setName("MiNiFi logging handler"); - return t; - } - }); + try { + RunMiNiFi runMiNiFi = new RunMiNiFi(BootstrapFileProvider.getBootstrapConfFile()); + System.exit(runMiNiFi.run(cmd.get(), args)); + } catch (Exception e) { + CMD_LOGGER.error("Exception happened during the bootstrap run, check logs for details"); + DEFAULT_LOGGER.error("", e); + System.exit(1); + } } private static void printUsage() { @@ -189,7 +145,7 @@ public class RunMiNiFi implements QueryableStatusAggregator, ConfigurationFileHo System.out.println("java org.apache.nifi.minifi.bootstrap.RunMiNiFi [options]"); System.out.println(); System.out.println("Valid commands include:"); - System.out.println(""); + System.out.println(); System.out.println("Start : Start a new instance of Apache MiNiFi"); System.out.println("Stop : Stop a running instance of Apache MiNiFi"); System.out.println("Restart : Stop Apache MiNiFi, if it is running, and then start a new instance"); @@ -200,1289 +156,38 @@ public class RunMiNiFi implements QueryableStatusAggregator, ConfigurationFileHo System.out.println(); } - public static void main(String[] args) throws IOException, InterruptedException { - if (args.length < 1 || args.length > 3) { - printUsage(); + public void setMiNiFiParameters(int port, String secretKey) throws IOException { + if (Optional.ofNullable(secretKey).filter(key -> key.equals(miNiFiParameters.getSecretKey())).isPresent() && miNiFiParameters.getMiNiFiPort() == port) { + DEFAULT_LOGGER.debug("secretKey and port match with the known one, nothing to update"); return; } - File dumpFile = null; + miNiFiParameters.setMiNiFiPort(port); + miNiFiParameters.setSecretKey(secretKey); - final String cmd = args[0]; - if (cmd.equals("dump")) { - if (args.length > 1) { - dumpFile = new File(args[1]); - } else { - dumpFile = null; - } - } - - switch (cmd.toLowerCase()) { - case "start": - case "run": - case "stop": - case "status": - case "dump": - case "restart": - case "env": - case "flowstatus": - break; - default: - printUsage(); - return; - } - - final File configFile = getBootstrapConfFile(); - final RunMiNiFi runMiNiFi = new RunMiNiFi(configFile); - - Integer exitStatus = null; - switch (cmd.toLowerCase()) { - case "start": - runMiNiFi.start(); - break; - case "run": - runMiNiFi.start(); - break; - case "stop": - runMiNiFi.stop(); - break; - case "status": - exitStatus = runMiNiFi.status(); - break; - case "restart": - runMiNiFi.stop(); - runMiNiFi.start(); - break; - case "dump": - runMiNiFi.dump(dumpFile); - break; - case "env": - runMiNiFi.env(); - break; - case "flowstatus": - if(args.length == 2) { - System.out.println(runMiNiFi.statusReport(args[1])); - } else { - System.out.println("The 'flowStatus' command requires an input query. See the System Admin Guide 'FlowStatus Script Query' section for complete details."); - } - break; - } - if (exitStatus != null) { - System.exit(exitStatus); - } - } - - public static File getBootstrapConfFile() { - String configFilename = System.getProperty("org.apache.nifi.minifi.bootstrap.config.file"); - - if (configFilename == null) { - final String nifiHome = System.getenv("MINIFI_HOME"); - if (nifiHome != null) { - final File nifiHomeFile = new File(nifiHome.trim()); - final File configFile = new File(nifiHomeFile, DEFAULT_CONFIG_FILE); - configFilename = configFile.getAbsolutePath(); - } - } - - if (configFilename == null) { - configFilename = DEFAULT_CONFIG_FILE; - } - - final File configFile = new File(configFilename); - return configFile; - } - - private File getBootstrapFile(final Logger logger, String directory, String defaultDirectory, String fileName) throws IOException { - - final File confDir = bootstrapConfigFile.getParentFile(); - final File nifiHome = confDir.getParentFile(); - - String confFileDir = System.getProperty(directory); - - final File fileDir; - - if (confFileDir != null) { - fileDir = new File(confFileDir.trim()); - } else { - fileDir = new File(nifiHome, defaultDirectory); - } - - FileUtils.ensureDirectoryExistAndCanAccess(fileDir); - final File statusFile = new File(fileDir, fileName); - logger.debug("Status File: {}", statusFile); - return statusFile; - } - - File getPidFile(final Logger logger) throws IOException { - return getBootstrapFile(logger, MINIFI_PID_DIR_PROP, DEFAULT_PID_DIR, MINIFI_PID_FILE_NAME); - } - - File getStatusFile(final Logger logger) throws IOException { - return getBootstrapFile(logger, MINIFI_PID_DIR_PROP, DEFAULT_PID_DIR, MINIFI_STATUS_FILE_NAME); - } - - File getLockFile(final Logger logger) throws IOException { - return getBootstrapFile(logger, MINIFI_PID_DIR_PROP, DEFAULT_PID_DIR, MINIFI_LOCK_FILE_NAME); - } - - File getStatusFile() throws IOException{ - return getStatusFile(defaultLogger); - } - - public File getReloadFile(final Logger logger) { - final File confDir = bootstrapConfigFile.getParentFile(); - final File nifiHome = confDir.getParentFile(); - final File bin = new File(nifiHome, "bin"); - final File reloadFile = new File(bin, "minifi.reload.lock"); - - logger.debug("Reload File: {}", reloadFile); - return reloadFile; - } - - public File getSwapFile(final Logger logger) { - final File confDir = bootstrapConfigFile.getParentFile(); - final File swapFile = new File(confDir, "swap.yml"); - - logger.debug("Swap File: {}", swapFile); - return swapFile; - } - - - private Properties loadProperties(final Logger logger) throws IOException { - final Properties props = new Properties(); - final File statusFile = getStatusFile(logger); - if (statusFile == null || !statusFile.exists()) { - logger.debug("No status file to load properties from"); - return props; - } - - try (final FileInputStream fis = new FileInputStream(getStatusFile(logger))) { - props.load(fis); - } - - final Map modified = new HashMap<>(props); - modified.remove("secret.key"); - logger.debug("Properties: {}", modified); - - return props; - } - - private synchronized void saveProperties(final Properties minifiProps, final Logger logger) throws IOException { - final String pid = minifiProps.getProperty(PID_KEY); - if (!StringUtils.isBlank(pid)) { - writePidFile(pid, logger); - } - - final File statusFile = getStatusFile(logger); - if (statusFile.exists() && !statusFile.delete()) { - logger.warn("Failed to delete {}", statusFile); - } - - if (!statusFile.createNewFile()) { - throw new IOException("Failed to create file " + statusFile); + Properties minifiProps = new Properties(); + long minifiPid = miNiFiParameters.getMinifiPid(); + if (minifiPid != UNINITIALIZED) { + minifiProps.setProperty(STATUS_FILE_PID_KEY, String.valueOf(minifiPid)); } + minifiProps.setProperty(STATUS_FILE_PORT_KEY, String.valueOf(port)); + minifiProps.setProperty(STATUS_FILE_SECRET_KEY, secretKey); + File statusFile = bootstrapFileProvider.getStatusFile(); try { - final Set perms = new HashSet<>(); - perms.add(PosixFilePermission.OWNER_WRITE); - perms.add(PosixFilePermission.OWNER_READ); - perms.add(PosixFilePermission.GROUP_READ); - perms.add(PosixFilePermission.OTHERS_READ); - Files.setPosixFilePermissions(statusFile.toPath(), perms); - } catch (final Exception e) { - logger.warn("Failed to set permissions so that only the owner can read status file {}; " - + "this may allows others to have access to the key needed to communicate with MiNiFi. " - + "Permissions should be changed so that only the owner can read this file", statusFile); + bootstrapFileProvider.saveStatusProperties(minifiProps); + } catch (IOException ioe) { + DEFAULT_LOGGER.warn("Apache MiNiFi has started but failed to persist MiNiFi Port information to {}", statusFile.getAbsolutePath(), ioe); } - try (final FileOutputStream fos = new FileOutputStream(statusFile)) { - minifiProps.store(fos, null); - fos.getFD().sync(); - } - - logger.debug("Saved Properties {} to {}", new Object[]{minifiProps, statusFile}); - } - - private synchronized void writePidFile(final String pid, final Logger logger) throws IOException { - final File pidFile = getPidFile(logger); - if (pidFile.exists() && !pidFile.delete()) { - logger.warn("Failed to delete {}", pidFile); - } - - if (!pidFile.createNewFile()) { - throw new IOException("Failed to create file " + pidFile); - } - - try { - final Set perms = new HashSet<>(); - perms.add(PosixFilePermission.OWNER_READ); - perms.add(PosixFilePermission.OWNER_WRITE); - Files.setPosixFilePermissions(pidFile.toPath(), perms); - } catch (final Exception e) { - logger.warn("Failed to set permissions so that only the owner can read pid file {}; " - + "this may allows others to have access to the key needed to communicate with MiNiFi. " - + "Permissions should be changed so that only the owner can read this file", pidFile); - } - - try (final FileOutputStream fos = new FileOutputStream(pidFile)) { - fos.write(pid.getBytes(StandardCharsets.UTF_8)); - fos.getFD().sync(); - } - - logger.debug("Saved Pid {} to {}", new Object[]{pid, pidFile}); - } - - private boolean isPingSuccessful(final int port, final String secretKey, final Logger logger) { - logger.debug("Pinging {}", port); - - try (final Socket socket = new Socket("localhost", port)) { - final OutputStream out = socket.getOutputStream(); - out.write((PING_CMD + " " + secretKey + "\n").getBytes(StandardCharsets.UTF_8)); - out.flush(); - - logger.debug("Sent PING command"); - socket.setSoTimeout(5000); - final InputStream in = socket.getInputStream(); - final BufferedReader reader = new BufferedReader(new InputStreamReader(in)); - final String response = reader.readLine(); - logger.debug("PING response: {}", response); - out.close(); - reader.close(); - - return PING_CMD.equals(response); - } catch (final IOException ioe) { - return false; - } - } - - private Integer getCurrentPort(final Logger logger) throws IOException { - final Properties props = loadProperties(logger); - final String portVal = props.getProperty("port"); - if (portVal == null) { - logger.debug("No Port found in status file"); - return null; - } else { - logger.debug("Port defined in status file: {}", portVal); - } - - final int port = Integer.parseInt(portVal); - final boolean success = isPingSuccessful(port, props.getProperty("secret.key"), logger); - if (success) { - logger.debug("Successful PING on port {}", port); - return port; - } - - final String pid = props.getProperty(PID_KEY); - logger.debug("PID in status file is {}", pid); - if (pid != null) { - final boolean procRunning = isProcessRunning(pid, logger); - if (procRunning) { - return port; - } else { - return null; - } - } - - return null; - } - - private boolean isProcessRunning(final String pid, final Logger logger) { - try { - // We use the "ps" command to check if the process is still running. - final ProcessBuilder builder = new ProcessBuilder(); - - builder.command("ps", "-p", pid); - final Process proc = builder.start(); - - // Look for the pid in the output of the 'ps' command. - boolean running = false; - String line; - try (final InputStream in = proc.getInputStream(); - final Reader streamReader = new InputStreamReader(in); - final BufferedReader reader = new BufferedReader(streamReader)) { - - while ((line = reader.readLine()) != null) { - if (line.trim().startsWith(pid)) { - running = true; - } - } - } - - // If output of the ps command had our PID, the process is running. - if (running) { - logger.debug("Process with PID {} is running", pid); - } else { - logger.debug("Process with PID {} is not running", pid); - } - - return running; - } catch (final IOException ioe) { - System.err.println("Failed to determine if Process " + pid + " is running; assuming that it is not"); - return false; - } - } - - private Status getStatus(final Logger logger) { - final Properties props; - try { - props = loadProperties(logger); - } catch (final IOException ioe) { - return new Status(null, null, false, false); - } - - if (props == null) { - return new Status(null, null, false, false); - } - - final String portValue = props.getProperty("port"); - final String pid = props.getProperty(PID_KEY); - final String secretKey = props.getProperty("secret.key"); - - if (portValue == null && pid == null) { - return new Status(null, null, false, false); - } - - Integer port = null; - boolean pingSuccess = false; - if (portValue != null) { - try { - port = Integer.parseInt(portValue); - pingSuccess = isPingSuccessful(port, secretKey, logger); - } catch (final NumberFormatException nfe) { - return new Status(null, null, false, false); - } - } - - if (pingSuccess) { - return new Status(port, pid, true, true); - } - - final boolean alive = (pid != null) && isProcessRunning(pid, logger); - return new Status(port, pid, pingSuccess, alive); - } - - public int status() throws IOException { - final Logger logger = cmdLogger; - final Status status = getStatus(logger); - if (status.isRespondingToPing()) { - logger.info("Apache MiNiFi is currently running, listening to Bootstrap on port {}, PID={}", - new Object[]{status.getPort(), status.getPid() == null ? "unknown" : status.getPid()}); - return 0; - } - - if (status.isProcessRunning()) { - logger.info("Apache MiNiFi is running at PID {} but is not responding to ping requests", status.getPid()); - return 4; - } - - if (status.getPort() == null) { - logger.info("Apache MiNiFi is not running"); - return 3; - } - - if (status.getPid() == null) { - logger.info("Apache MiNiFi is not responding to Ping requests. The process may have died or may be hung"); - } else { - logger.info("Apache MiNiFi is not running"); - } - return 3; - } - - public FlowStatusReport statusReport(String statusRequest) throws IOException { - final Logger logger = cmdLogger; - final Status status = getStatus(logger); - final Properties props = loadProperties(logger); - - List problemsGeneratingReport = new LinkedList<>(); - if (!status.isProcessRunning()) { - problemsGeneratingReport.add("MiNiFi process is not running"); - } - - if (!status.isRespondingToPing()) { - problemsGeneratingReport.add("MiNiFi process is not responding to pings"); - } - - if (!problemsGeneratingReport.isEmpty()) { - FlowStatusReport flowStatusReport = new FlowStatusReport(); - flowStatusReport.setErrorsGeneratingReport(problemsGeneratingReport); - return flowStatusReport; - } - - return getFlowStatusReport(statusRequest, status.getPort(), props.getProperty("secret.key"), logger); - } - - public void env() { - final Logger logger = cmdLogger; - final Status status = getStatus(logger); - if (status.getPid() == null) { - logger.info("Apache MiNiFi is not running"); - return; - } - final Class virtualMachineClass; - try { - virtualMachineClass = Class.forName("com.sun.tools.attach.VirtualMachine"); - } catch (final ClassNotFoundException cnfe) { - logger.error("Seems tools.jar (Linux / Windows JDK) or classes.jar (Mac OS) is not available in classpath"); - return; - } - final Method attachMethod; - final Method detachMethod; - - try { - attachMethod = virtualMachineClass.getMethod("attach", String.class); - detachMethod = virtualMachineClass.getDeclaredMethod("detach"); - } catch (final Exception e) { - logger.error("Methods required for getting environment not available", e); - return; - } - - final Object virtualMachine; - try { - virtualMachine = attachMethod.invoke(null, status.getPid()); - } catch (final Throwable t) { - logger.error("Problem attaching to MiNiFi", t); - return; - } - - try { - final Method getSystemPropertiesMethod = virtualMachine.getClass().getMethod("getSystemProperties"); - - final Properties sysProps = (Properties) getSystemPropertiesMethod.invoke(virtualMachine); - for (Entry syspropEntry : sysProps.entrySet()) { - logger.info(syspropEntry.getKey().toString() + " = " + syspropEntry.getValue().toString()); - } - } catch (Throwable t) { - throw new RuntimeException(t); - } finally { - try { - detachMethod.invoke(virtualMachine); - } catch (final Exception e) { - logger.warn("Caught exception detaching from process", e); - } - } - } - - /** - * Writes a MiNiFi thread dump to the given file; if file is null, logs at - * INFO level instead. - * - * @param dumpFile the file to write the dump content to - * @throws IOException if any issues occur while writing the dump file - */ - public void dump(final File dumpFile) throws IOException { - final Logger logger = defaultLogger; // dump to bootstrap log file by default - final Integer port = getCurrentPort(logger); - if (port == null) { - logger.info("Apache MiNiFi is not currently running"); - return; - } - - final Properties minifiProps = loadProperties(logger); - final String secretKey = minifiProps.getProperty("secret.key"); - - final StringBuilder sb = new StringBuilder(); - try (final Socket socket = new Socket()) { - logger.debug("Connecting to MiNiFi instance"); - socket.setSoTimeout(60000); - socket.connect(new InetSocketAddress("localhost", port)); - logger.debug("Established connection to MiNiFi instance."); - socket.setSoTimeout(60000); - - logger.debug("Sending DUMP Command to port {}", port); - final OutputStream out = socket.getOutputStream(); - out.write((DUMP_CMD + " " + secretKey + "\n").getBytes(StandardCharsets.UTF_8)); - out.flush(); - - final InputStream in = socket.getInputStream(); - try (final BufferedReader reader = new BufferedReader(new InputStreamReader(in))) { - String line; - while ((line = reader.readLine()) != null) { - sb.append(line).append("\n"); - } - } - } - - final String dump = sb.toString(); - if (dumpFile == null) { - logger.info(dump); - } else { - try (final FileOutputStream fos = new FileOutputStream(dumpFile)) { - fos.write(dump.getBytes(StandardCharsets.UTF_8)); - } - // we want to log to the console (by default) that we wrote the thread dump to the specified file - cmdLogger.info("Successfully wrote thread dump to {}", dumpFile.getAbsolutePath()); - } + CMD_LOGGER.info("The thread to run Apache MiNiFi is now running and listening for Bootstrap requests on port {}", port); } public void reload() throws IOException { - final Logger logger = defaultLogger; - final Integer port = getCurrentPort(logger); - if (port == null) { - logger.info("Apache MiNiFi is not currently running"); - return; - } - - // indicate that a reload command is in progress - final File reloadLockFile = getReloadFile(logger); - if (!reloadLockFile.exists()) { - reloadLockFile.createNewFile(); - } - - final Properties minifiProps = loadProperties(logger); - final String secretKey = minifiProps.getProperty("secret.key"); - final String pid = minifiProps.getProperty(PID_KEY); - - try (final Socket socket = new Socket()) { - logger.debug("Connecting to MiNiFi instance"); - socket.setSoTimeout(10000); - socket.connect(new InetSocketAddress("localhost", port)); - logger.debug("Established connection to MiNiFi instance."); - socket.setSoTimeout(10000); - - logger.debug("Sending RELOAD Command to port {}", port); - final OutputStream out = socket.getOutputStream(); - out.write((RELOAD_CMD + " " + secretKey + "\n").getBytes(StandardCharsets.UTF_8)); - out.flush(); - socket.shutdownOutput(); - - final InputStream in = socket.getInputStream(); - int lastChar; - final StringBuilder sb = new StringBuilder(); - while ((lastChar = in.read()) > -1) { - sb.append((char) lastChar); - } - final String response = sb.toString().trim(); - - logger.debug("Received response to RELOAD command: {}", response); - - if (RELOAD_CMD.equals(response)) { - logger.info("Apache MiNiFi has accepted the Reload Command and is reloading"); - - if (pid != null) { - final Properties bootstrapProperties = getBootstrapProperties(); - - String gracefulShutdown = bootstrapProperties.getProperty(GRACEFUL_SHUTDOWN_PROP, DEFAULT_GRACEFUL_SHUTDOWN_VALUE); - int gracefulShutdownSeconds; - try { - gracefulShutdownSeconds = Integer.parseInt(gracefulShutdown); - } catch (final NumberFormatException nfe) { - gracefulShutdownSeconds = Integer.parseInt(DEFAULT_GRACEFUL_SHUTDOWN_VALUE); - } - - final long startWait = System.nanoTime(); - while (isProcessRunning(pid, logger)) { - logger.info("Waiting for Apache MiNiFi to finish shutting down..."); - final long waitNanos = System.nanoTime() - startWait; - final long waitSeconds = TimeUnit.NANOSECONDS.toSeconds(waitNanos); - if (waitSeconds >= gracefulShutdownSeconds && gracefulShutdownSeconds > 0) { - if (isProcessRunning(pid, logger)) { - logger.warn("MiNiFi has not finished shutting down after {} seconds as part of configuration reload. Killing process.", gracefulShutdownSeconds); - try { - killProcessTree(pid, logger); - } catch (final IOException ioe) { - logger.error("Failed to kill Process with PID {}", pid); - } - } - break; - } else { - try { - Thread.sleep(2000L); - } catch (final InterruptedException ie) { - } - } - } - - reloading.set(true); - logger.info("MiNiFi has finished shutting down and will be reloaded."); - } - } else { - logger.error("When sending RELOAD command to MiNiFi, got unexpected response {}", response); - } - } catch (final IOException ioe) { - if (pid == null) { - logger.error("Failed to send shutdown command to port {} due to {}. No PID found for the MiNiFi process, so unable to kill process; " - + "the process should be killed manually.", new Object[]{port, ioe.toString()}); - } else { - logger.error("Failed to send shutdown command to port {} due to {}. Will kill the MiNiFi Process with PID {}.", port, ioe.toString(), pid); - killProcessTree(pid, logger); - } - } finally { - if (reloadLockFile.exists() && !reloadLockFile.delete()) { - logger.error("Failed to delete reload lock file {}; this file should be cleaned up manually", reloadLockFile); - } - } + reloadService.reload(); } - public void stop() throws IOException { - final Logger logger = cmdLogger; - final Integer port = getCurrentPort(logger); - if (port == null) { - logger.info("Apache MiNiFi is not currently running"); - return; - } - - // indicate that a stop command is in progress - final File lockFile = getLockFile(logger); - if (!lockFile.exists()) { - lockFile.createNewFile(); - } - - final Properties minifiProps = loadProperties(logger); - final String secretKey = minifiProps.getProperty("secret.key"); - final String pid = minifiProps.getProperty(PID_KEY); - final File statusFile = getStatusFile(logger); - final File pidFile = getPidFile(logger); - - try (final Socket socket = new Socket()) { - logger.debug("Connecting to MiNiFi instance"); - socket.setSoTimeout(10000); - socket.connect(new InetSocketAddress("localhost", port)); - logger.debug("Established connection to MiNiFi instance."); - socket.setSoTimeout(10000); - - logger.debug("Sending SHUTDOWN Command to port {}", port); - final OutputStream out = socket.getOutputStream(); - out.write((SHUTDOWN_CMD + " " + secretKey + "\n").getBytes(StandardCharsets.UTF_8)); - out.flush(); - socket.shutdownOutput(); - - final InputStream in = socket.getInputStream(); - int lastChar; - final StringBuilder sb = new StringBuilder(); - while ((lastChar = in.read()) > -1) { - sb.append((char) lastChar); - } - final String response = sb.toString().trim(); - - logger.debug("Received response to SHUTDOWN command: {}", response); - - if (SHUTDOWN_CMD.equals(response)) { - logger.info("Apache MiNiFi has accepted the Shutdown Command and is shutting down now"); - - if (pid != null) { - final Properties bootstrapProperties = getBootstrapProperties(); - - String gracefulShutdown = bootstrapProperties.getProperty(GRACEFUL_SHUTDOWN_PROP, DEFAULT_GRACEFUL_SHUTDOWN_VALUE); - int gracefulShutdownSeconds; - try { - gracefulShutdownSeconds = Integer.parseInt(gracefulShutdown); - } catch (final NumberFormatException nfe) { - gracefulShutdownSeconds = Integer.parseInt(DEFAULT_GRACEFUL_SHUTDOWN_VALUE); - } - - final long startWait = System.nanoTime(); - while (isProcessRunning(pid, logger)) { - logger.info("Waiting for Apache MiNiFi to finish shutting down..."); - final long waitNanos = System.nanoTime() - startWait; - final long waitSeconds = TimeUnit.NANOSECONDS.toSeconds(waitNanos); - if (waitSeconds >= gracefulShutdownSeconds && gracefulShutdownSeconds > 0) { - if (isProcessRunning(pid, logger)) { - logger.warn("MiNiFi has not finished shutting down after {} seconds. Killing process.", gracefulShutdownSeconds); - try { - killProcessTree(pid, logger); - } catch (final IOException ioe) { - logger.error("Failed to kill Process with PID {}", pid); - } - } - break; - } else { - try { - Thread.sleep(2000L); - } catch (final InterruptedException ie) { - } - } - } - - if (statusFile.exists() && !statusFile.delete()) { - logger.error("Failed to delete status file {}; this file should be cleaned up manually", statusFile); - } - - if (pidFile.exists() && !pidFile.delete()) { - logger.error("Failed to delete pid file {}; this file should be cleaned up manually", pidFile); - } - - logger.info("MiNiFi has finished shutting down."); - } - } else { - logger.error("When sending SHUTDOWN command to MiNiFi, got unexpected response {}", response); - } - } catch (final IOException ioe) { - if (pid == null) { - logger.error("Failed to send shutdown command to port {} due to {}. No PID found for the MiNiFi process, so unable to kill process; " - + "the process should be killed manually.", new Object[]{port, ioe.toString()}); - } else { - logger.error("Failed to send shutdown command to port {} due to {}. Will kill the MiNiFi Process with PID {}.", new Object[]{port, ioe.toString(), pid}); - killProcessTree(pid, logger); - if (statusFile.exists() && !statusFile.delete()) { - logger.error("Failed to delete status file {}; this file should be cleaned up manually", statusFile); - } - } - } finally { - if (lockFile.exists() && !lockFile.delete()) { - logger.error("Failed to delete lock file {}; this file should be cleaned up manually", lockFile); - } - } - } - - private Properties getBootstrapProperties() throws IOException { - final Properties bootstrapProperties = new Properties(); - try (final FileInputStream fis = new FileInputStream(bootstrapConfigFile)) { - bootstrapProperties.load(fis); - } - return bootstrapProperties; - } - - private static List getChildProcesses(final String ppid) throws IOException { - final Process proc = Runtime.getRuntime().exec(new String[]{"ps", "-o", PID_KEY, "--no-headers", "--ppid", ppid}); - final List childPids = new ArrayList<>(); - try (final InputStream in = proc.getInputStream(); - final BufferedReader reader = new BufferedReader(new InputStreamReader(in))) { - - String line; - while ((line = reader.readLine()) != null) { - childPids.add(line.trim()); - } - } - - return childPids; - } - - private void killProcessTree(final String pid, final Logger logger) throws IOException { - logger.debug("Killing Process Tree for PID {}", pid); - - final List children = getChildProcesses(pid); - logger.debug("Children of PID {}: {}", new Object[]{pid, children}); - - for (final String childPid : children) { - killProcessTree(childPid, logger); - } - - Runtime.getRuntime().exec(new String[]{"kill", "-9", pid}); - } - - public static boolean isAlive(final Process process) { - try { - process.exitValue(); - return false; - } catch (final IllegalStateException | IllegalThreadStateException itse) { - return true; - } - } - - private String getHostname() { - String hostname = "Unknown Host"; - String ip = "Unknown IP Address"; - try { - final InetAddress localhost = InetAddress.getLocalHost(); - hostname = localhost.getHostName(); - ip = localhost.getHostAddress(); - } catch (final Exception e) { - defaultLogger.warn("Failed to obtain hostname for notification due to:", e); - } - - return hostname + " (" + ip + ")"; - } - - private int getGracefulShutdownSeconds(Map props, File bootstrapConfigAbsoluteFile) { - String gracefulShutdown = props.get(GRACEFUL_SHUTDOWN_PROP); - if (gracefulShutdown == null) { - gracefulShutdown = DEFAULT_GRACEFUL_SHUTDOWN_VALUE; - } - - final int gracefulShutdownSeconds; - try { - gracefulShutdownSeconds = Integer.parseInt(gracefulShutdown); - } catch (final NumberFormatException nfe) { - throw new NumberFormatException("The '" + GRACEFUL_SHUTDOWN_PROP + "' property in Bootstrap Config File " - + bootstrapConfigAbsoluteFile.getAbsolutePath() + " has an invalid value. Must be a non-negative integer"); - } - - if (gracefulShutdownSeconds < 0) { - throw new NumberFormatException("The '" + GRACEFUL_SHUTDOWN_PROP + "' property in Bootstrap Config File " - + bootstrapConfigAbsoluteFile.getAbsolutePath() + " has an invalid value. Must be a non-negative integer"); - } - return gracefulShutdownSeconds; - } - - private Map readProperties() throws IOException { - if (!bootstrapConfigFile.exists()) { - throw new FileNotFoundException(bootstrapConfigFile.getAbsolutePath()); - } - - final Properties properties = new Properties(); - try (final FileInputStream fis = new FileInputStream(bootstrapConfigFile)) { - properties.load(fis); - } - - final Map props = new HashMap<>(); - props.putAll((Map) properties); - return props; - } - - @SuppressWarnings({"rawtypes", "unchecked"}) - public Tuple startMiNiFi() throws IOException, InterruptedException { - final Integer port = getCurrentPort(cmdLogger); - if (port != null) { - cmdLogger.info("Apache MiNiFi is already running, listening to Bootstrap on port " + port); - return null; - } - - final File prevLockFile = getLockFile(cmdLogger); - if (prevLockFile.exists() && !prevLockFile.delete()) { - cmdLogger.warn("Failed to delete previous lock file {}; this file should be cleaned up manually", prevLockFile); - } - - final ProcessBuilder builder = new ProcessBuilder(); - - final Map props = readProperties(); - - final String specifiedWorkingDir = props.get("working.dir"); - if (specifiedWorkingDir != null) { - builder.directory(new File(specifiedWorkingDir)); - } - - final File bootstrapConfigAbsoluteFile = bootstrapConfigFile.getAbsoluteFile(); - final File binDir = bootstrapConfigAbsoluteFile.getParentFile(); - final File workingDir = binDir.getParentFile(); - - if (specifiedWorkingDir == null) { - builder.directory(workingDir); - } - - final String minifiLogDir = replaceNull(System.getProperty("org.apache.nifi.minifi.bootstrap.config.log.dir"), DEFAULT_LOG_DIR).trim(); - - final String libFilename = replaceNull(props.get("lib.dir"), "./lib").trim(); - File libDir = getFile(libFilename, workingDir); - - final String confFilename = replaceNull(props.get(CONF_DIR_KEY), "./conf").trim(); - File confDir = getFile(confFilename, workingDir); - - String minifiPropsFilename = props.get("props.file"); - if (minifiPropsFilename == null) { - if (confDir.exists()) { - minifiPropsFilename = new File(confDir, "nifi.properties").getAbsolutePath(); - } else { - minifiPropsFilename = DEFAULT_CONFIG_FILE; - } - } - - minifiPropsFilename = minifiPropsFilename.trim(); - - final List javaAdditionalArgs = new ArrayList<>(); - for (final Entry entry : props.entrySet()) { - final String key = entry.getKey(); - final String value = entry.getValue(); - - if (key.startsWith("java.arg")) { - javaAdditionalArgs.add(value); - } - } - - final File[] libFiles = libDir.listFiles(new FilenameFilter() { - @Override - public boolean accept(final File dir, final String filename) { - return filename.toLowerCase().endsWith(".jar"); - } - }); - - if (libFiles == null || libFiles.length == 0) { - throw new RuntimeException("Could not find lib directory at " + libDir.getAbsolutePath()); - } - - final File[] confFiles = confDir.listFiles(); - if (confFiles == null || confFiles.length == 0) { - throw new RuntimeException("Could not find conf directory at " + confDir.getAbsolutePath()); - } - - final List cpFiles = new ArrayList<>(confFiles.length + libFiles.length); - cpFiles.add(confDir.getAbsolutePath()); - for (final File file : libFiles) { - cpFiles.add(file.getAbsolutePath()); - } - - final StringBuilder classPathBuilder = new StringBuilder(); - for (int i = 0; i < cpFiles.size(); i++) { - final String filename = cpFiles.get(i); - classPathBuilder.append(filename); - if (i < cpFiles.size() - 1) { - classPathBuilder.append(File.pathSeparatorChar); - } - } - - final String classPath = classPathBuilder.toString(); - String javaCmd = props.get("java"); - if (javaCmd == null) { - javaCmd = DEFAULT_JAVA_CMD; - } - if (javaCmd.equals(DEFAULT_JAVA_CMD)) { - String javaHome = System.getenv("JAVA_HOME"); - if (javaHome != null) { - String fileExtension = isWindows() ? ".exe" : ""; - File javaFile = new File(javaHome + File.separatorChar + "bin" - + File.separatorChar + "java" + fileExtension); - if (javaFile.exists() && javaFile.canExecute()) { - javaCmd = javaFile.getAbsolutePath(); - } - } - } - - final MiNiFiListener listener = new MiNiFiListener(); - final int listenPort = listener.start(this); - - final List cmd = new ArrayList<>(); - - cmd.add(javaCmd); - cmd.add("-classpath"); - cmd.add(classPath); - cmd.addAll(javaAdditionalArgs); - cmd.add("-Dnifi.properties.file.path=" + minifiPropsFilename); - cmd.add("-Dnifi.bootstrap.listen.port=" + listenPort); - cmd.add("-Dapp=MiNiFi"); - cmd.add("-Dorg.apache.nifi.minifi.bootstrap.config.log.dir="+minifiLogDir); - cmd.add("org.apache.nifi.minifi.MiNiFi"); - - builder.command(cmd); - - final StringBuilder cmdBuilder = new StringBuilder(); - for (final String s : cmd) { - cmdBuilder.append(s).append(" "); - } - - cmdLogger.info("Starting Apache MiNiFi..."); - cmdLogger.info("Working Directory: {}", workingDir.getAbsolutePath()); - cmdLogger.info("Command: {}", cmdBuilder.toString()); - - - Process process = builder.start(); - handleLogging(process); - Long pid = OSUtils.getProcessId(process, cmdLogger); - if (pid != null) { - minifiPid = pid; - final Properties minifiProps = new Properties(); - minifiProps.setProperty(PID_KEY, String.valueOf(minifiPid)); - saveProperties(minifiProps, cmdLogger); - } - - gracefulShutdownSeconds = getGracefulShutdownSeconds(props, bootstrapConfigAbsoluteFile); - shutdownHook = new ShutdownHook(process, this, secretKey, gracefulShutdownSeconds, loggingExecutor); - final Runtime runtime = Runtime.getRuntime(); - runtime.addShutdownHook(shutdownHook); - - return new Tuple(builder, process); - } - - @SuppressWarnings({"rawtypes", "unchecked"}) - public void start() throws IOException, InterruptedException { - - final String confDir = getBootstrapProperties().getProperty(CONF_DIR_KEY); - final File configFile = new File(getBootstrapProperties().getProperty(MINIFI_CONFIG_FILE_KEY)); - try (InputStream inputStream = new FileInputStream(configFile)) { - ByteBuffer tempConfigFile = performTransformation(inputStream, confDir); - currentConfigFileReference.set(tempConfigFile.asReadOnlyBuffer()); - } catch (ConfigurationChangeException e) { - defaultLogger.error("The config file is malformed, unable to start.", e); - return; - } - - // Instantiate configuration listener and configured ingestors - this.changeListener = new MiNiFiConfigurationChangeListener(this, defaultLogger); - this.periodicStatusReporters = initializePeriodicNotifiers(); - startPeriodicNotifiers(); - try { - this.changeCoordinator = initializeNotifier(this.changeListener); - } catch (Exception e) { - final String errorMsg = "Unable to start as {} is not properly configured due to: {}"; - cmdLogger.error(errorMsg, this.changeListener.getDescriptor(), e.getMessage()); - defaultLogger.error("Unable to initialize notifier.", e); - // if we fail to initialize, exit without attempting to start - System.exit(1); - } - - Tuple tuple = startMiNiFi(); - if (tuple == null) { - cmdLogger.info("Start method returned null, ending start command."); - return; - } - - ProcessBuilder builder = tuple.getKey(); - Process process = tuple.getValue(); - - try { - while (true) { - final boolean alive = isAlive(process); - - if (alive) { - try { - Thread.sleep(1000L); - - if (reloading.get() && getNifiStarted()) { - final File swapConfigFile = getSwapFile(defaultLogger); - if (swapConfigFile.exists()) { - defaultLogger.info("MiNiFi has finished reloading successfully and swap file exists. Deleting old configuration."); - - if (swapConfigFile.delete()) { - defaultLogger.info("Swap file was successfully deleted."); - } else { - defaultLogger.error("Swap file was not deleted. It should be deleted manually."); - } - } - - reloading.set(false); - } - - } catch (final InterruptedException ie) { - } - } else { - final Runtime runtime = Runtime.getRuntime(); - try { - runtime.removeShutdownHook(shutdownHook); - } catch (final IllegalStateException ise) { - // happens when already shutting down - } - - if (autoRestartNiFi) { - final File statusFile = getStatusFile(defaultLogger); - if (!statusFile.exists()) { - defaultLogger.info("Status File no longer exists. Will not restart MiNiFi"); - return; - } - - final File lockFile = getLockFile(defaultLogger); - if (lockFile.exists()) { - defaultLogger.info("A shutdown was initiated. Will not restart MiNiFi"); - return; - } - - final File reloadFile = getReloadFile(defaultLogger); - if (reloadFile.exists()) { - defaultLogger.info("Currently reloading configuration. Will wait to restart MiNiFi."); - Thread.sleep(5000L); - continue; - } - - final boolean previouslyStarted = getNifiStarted(); - if (!previouslyStarted) { - final File swapConfigFile = getSwapFile(defaultLogger); - if (swapConfigFile.exists()) { - defaultLogger.info("Swap file exists, MiNiFi failed trying to change configuration. Reverting to old configuration."); - - try { - ByteBuffer tempConfigFile = performTransformation(new FileInputStream(swapConfigFile), confDir); - currentConfigFileReference.set(tempConfigFile.asReadOnlyBuffer()); - } catch (ConfigurationChangeException e) { - defaultLogger.error("The swap file is malformed, unable to restart from prior state. Will not attempt to restart MiNiFi. Swap File should be cleaned up manually."); - return; - } - - Files.copy(swapConfigFile.toPath(), Paths.get(getBootstrapProperties().getProperty(MINIFI_CONFIG_FILE_KEY)), REPLACE_EXISTING); - - defaultLogger.info("Replacing config file with swap file and deleting swap file"); - if (!swapConfigFile.delete()) { - defaultLogger.warn("The swap file failed to delete after replacing using it to revert to the old configuration. It should be cleaned up manually."); - } - reloading.set(false); - } else { - defaultLogger.info("MiNiFi either never started or failed to restart. Will not attempt to restart MiNiFi"); - return; - } - } else { - setNiFiStarted(false); - } - - secretKey = null; - process = builder.start(); - handleLogging(process); - - Long pid = OSUtils.getProcessId(process, defaultLogger); - if (pid != null) { - minifiPid = pid; - final Properties minifiProps = new Properties(); - minifiProps.setProperty(PID_KEY, String.valueOf(minifiPid)); - saveProperties(minifiProps, defaultLogger); - } - - shutdownHook = new ShutdownHook(process, this, secretKey, gracefulShutdownSeconds, loggingExecutor); - runtime.addShutdownHook(shutdownHook); - - final boolean started = waitForStart(); - - if (started) { - defaultLogger.info("Successfully spawned the thread to start Apache MiNiFi{}", (pid == null ? "" : " with PID " + pid)); - } else { - defaultLogger.error("Apache MiNiFi does not appear to have started"); - } - } else { - return; - } - } - } - } finally { - shutdownChangeNotifier(); - shutdownPeriodicStatusReporters(); - } - } - - public FlowStatusReport getFlowStatusReport(String statusRequest, final int port, final String secretKey, final Logger logger) throws IOException { - logger.debug("Pinging {}", port); - - try (final Socket socket = new Socket("localhost", port)) { - final OutputStream out = socket.getOutputStream(); - final String commandWithArgs = FLOW_STATUS_REPORT_CMD + " " + secretKey +" " + statusRequest + "\n"; - out.write((commandWithArgs).getBytes(StandardCharsets.UTF_8)); - logger.debug("Sending command to MiNiFi: {}",commandWithArgs); - out.flush(); - - logger.debug("Sent FLOW_STATUS_REPORT_CMD to MiNiFi"); - socket.setSoTimeout(5000); - final InputStream in = socket.getInputStream(); - - ObjectInputStream ois = new ObjectInputStream(in); - logger.debug("FLOW_STATUS_REPORT_CMD response received"); - Object o = ois.readObject(); - ois.close(); - out.close(); - try { - return FlowStatusReport.class.cast(o); - } catch (ClassCastException e) { - String message = String.class.cast(o); - FlowStatusReport flowStatusReport = new FlowStatusReport(); - flowStatusReport.setErrorsGeneratingReport(Collections.singletonList("Failed to get status report from MiNiFi due to:" + message)); - return flowStatusReport; - } - } catch (EOFException | ClassNotFoundException | SocketTimeoutException e) { - throw new IllegalStateException("Failed to get the status report from the MiNiFi process. Potentially due to the process currently being down (restarting or otherwise).", e); - } - } - - private void handleLogging(final Process process) { - final Set> existingFutures = loggingFutures; - if (existingFutures != null) { - for (final Future future : existingFutures) { - future.cancel(false); - } - } - - final Future stdOutFuture = loggingExecutor.submit(new Runnable() { - @Override - public void run() { - final Logger stdOutLogger = LoggerFactory.getLogger("org.apache.nifi.minifi.StdOut"); - final InputStream in = process.getInputStream(); - try (final BufferedReader reader = new BufferedReader(new InputStreamReader(in))) { - String line; - while ((line = reader.readLine()) != null) { - stdOutLogger.info(line); - } - } catch (IOException e) { - defaultLogger.error("Failed to read from MiNiFi's Standard Out stream", e); - } - } - }); - - final Future stdErrFuture = loggingExecutor.submit(new Runnable() { - @Override - public void run() { - final Logger stdErrLogger = LoggerFactory.getLogger("org.apache.nifi.minifi.StdErr"); - final InputStream in = process.getErrorStream(); - try (final BufferedReader reader = new BufferedReader(new InputStreamReader(in))) { - String line; - while ((line = reader.readLine()) != null) { - stdErrLogger.error(line); - } - } catch (IOException e) { - defaultLogger.error("Failed to read from MiNiFi's Standard Error stream", e); - } - } - }); - - final Set> futures = new HashSet<>(); - futures.add(stdOutFuture); - futures.add(stdErrFuture); - this.loggingFutures = futures; - } - - private boolean isWindows() { - final String osName = System.getProperty("os.name"); - return osName != null && osName.toLowerCase().contains("win"); - } - - private boolean waitForStart() { - lock.lock(); - try { - final long startTime = System.nanoTime(); - - while (ccPort < 1) { - try { - startupCondition.await(1, TimeUnit.SECONDS); - } catch (final InterruptedException ie) { - return false; - } - - final long waitNanos = System.nanoTime() - startTime; - final long waitSeconds = TimeUnit.NANOSECONDS.toSeconds(waitNanos); - if (waitSeconds > STARTUP_WAIT_SECONDS) { - return false; - } - } - } finally { - lock.unlock(); - } - return true; - } - - private File getFile(final String filename, final File workingDir) { - File file = new File(filename); - if (!file.isAbsolute()) { - file = new File(workingDir, filename); - } - - return file; - } - - private String replaceNull(final String value, final String replacement) { - return (value == null) ? replacement : value; - } - - void setAutoRestartNiFi(final boolean restart) { - this.autoRestartNiFi = restart; - } - - void setMiNiFiCommandControlPort(final int port, final String secretKey) throws IOException { - - if (this.secretKey != null && this.ccPort != UNINITIALIZED_CC_PORT) { - defaultLogger.warn("Blocking attempt to change MiNiFi command port and secret after they have already been initialized. requestedPort={}", port); - return; - } - - this.ccPort = port; - this.secretKey = secretKey; - - if (shutdownHook != null) { - shutdownHook.setSecretKey(secretKey); - } - - final File statusFile = getStatusFile(defaultLogger); - - final Properties minifiProps = new Properties(); - if (minifiPid != -1) { - minifiProps.setProperty(PID_KEY, String.valueOf(minifiPid)); - } - minifiProps.setProperty("port", String.valueOf(ccPort)); - minifiProps.setProperty("secret.key", secretKey); - - try { - saveProperties(minifiProps, defaultLogger); - } catch (final IOException ioe) { - defaultLogger.warn("Apache MiNiFi has started but failed to persist MiNiFi Port information to {} due to {}", new Object[]{statusFile.getAbsolutePath(), ioe}); - } - - defaultLogger.info("The thread to run Apache MiNiFi is now running and listening for Bootstrap requests on port {}", port); - } - - int getNiFiCommandControlPort() { - return this.ccPort; - } - - void setNiFiStarted(final boolean nifiStarted) { + public void setNiFiStarted(boolean nifiStarted) { startedLock.lock(); try { this.nifiStarted = nifiStarted; @@ -1491,248 +196,53 @@ public class RunMiNiFi implements QueryableStatusAggregator, ConfigurationFileHo } } - boolean getNifiStarted() { + public boolean isNiFiStarted() { startedLock.lock(); try { - return nifiStarted; + return this.nifiStarted; } finally { startedLock.unlock(); } } public void shutdownChangeNotifier() { - try { - getChangeCoordinator().close(); - } catch (IOException e) { - defaultLogger.warn("Could not successfully stop notifier ", e); - } + configurationChangeCoordinator.close(); } - public ConfigurationChangeCoordinator getChangeCoordinator() { - return changeCoordinator; + public PeriodicStatusReporterManager getPeriodicStatusReporterManager() { + return periodicStatusReporterManager; } - private ConfigurationChangeCoordinator initializeNotifier(ConfigurationChangeListener configChangeListener) throws IOException { - final Properties bootstrapProperties = getBootstrapProperties(); - - ConfigurationChangeCoordinator notifier = new ConfigurationChangeCoordinator(); - notifier.initialize(bootstrapProperties, this, Collections.singleton(configChangeListener)); - notifier.start(); - - return notifier; + public ConfigurationChangeCoordinator getConfigurationChangeCoordinator() { + return configurationChangeCoordinator; } - public Set getPeriodicStatusReporters() { - return Collections.unmodifiableSet(periodicStatusReporters); + void setAutoRestartNiFi(boolean restart) { + this.autoRestartNiFi = restart; } - public void shutdownPeriodicStatusReporters() { - for (PeriodicStatusReporter periodicStatusReporter : getPeriodicStatusReporters()) { - try { - periodicStatusReporter.stop(); - } catch (Exception exception) { - System.out.println("Could not successfully stop periodic status reporter " + periodicStatusReporter.getClass() + " due to " + exception); - } - } + public Boolean isAutoRestartNiFi() { + return autoRestartNiFi; } - private Set initializePeriodicNotifiers() throws IOException { - final Set statusReporters = new HashSet<>(); - - final Properties bootstrapProperties = getBootstrapProperties(); - - final String reportersCsv = bootstrapProperties.getProperty(STATUS_REPORTER_COMPONENTS_KEY); - if (reportersCsv != null && !reportersCsv.isEmpty()) { - for (String reporterClassname : Arrays.asList(reportersCsv.split(","))) { - try { - Class reporterClass = Class.forName(reporterClassname); - PeriodicStatusReporter reporter = (PeriodicStatusReporter) reporterClass.newInstance(); - reporter.initialize(bootstrapProperties, this); - statusReporters.add(reporter); - } catch (InstantiationException | IllegalAccessException | ClassNotFoundException e) { - throw new RuntimeException("Issue instantiating notifier " + reporterClassname, e); - } - } - } - return statusReporters; + public boolean getReloading() { + return reloading.get(); } - private void startPeriodicNotifiers() throws IOException { - for (PeriodicStatusReporter periodicStatusReporter: this.periodicStatusReporters) { - periodicStatusReporter.start(); - } + public void setReloading(boolean val) { + reloading.set(val); } - private static class MiNiFiConfigurationChangeListener implements ConfigurationChangeListener { - - private final RunMiNiFi runner; - private final Logger logger; - private static final ReentrantLock handlingLock = new ReentrantLock(); - - public MiNiFiConfigurationChangeListener(RunMiNiFi runner, Logger logger) { - this.runner = runner; - this.logger = logger; - } - - @Override - public void handleChange(InputStream configInputStream) throws ConfigurationChangeException { - logger.info("Received notification of a change"); - - if (!handlingLock.tryLock()) { - throw new ConfigurationChangeException("Instance is already handling another change"); - } - try { - - final Properties bootstrapProperties = runner.getBootstrapProperties(); - final File configFile = new File(bootstrapProperties.getProperty(MINIFI_CONFIG_FILE_KEY)); - - // Store the incoming stream as a byte array to be shared among components that need it - final ByteArrayOutputStream bufferedConfigOs = new ByteArrayOutputStream(); - byte[] copyArray = new byte[1024]; - int available = -1; - while ((available = configInputStream.read(copyArray)) > 0) { - bufferedConfigOs.write(copyArray, 0, available); - } - - // Create an input stream to use for writing a config file as well as feeding to the config transformer - try (final ByteArrayInputStream newConfigBais = new ByteArrayInputStream(bufferedConfigOs.toByteArray())) { - newConfigBais.mark(-1); - - final File swapConfigFile = runner.getSwapFile(logger); - logger.info("Persisting old configuration to {}", swapConfigFile.getAbsolutePath()); - - try (FileInputStream configFileInputStream = new FileInputStream(configFile)) { - Files.copy(configFileInputStream, swapConfigFile.toPath(), REPLACE_EXISTING); - } - - try { - logger.info("Persisting changes to {}", configFile.getAbsolutePath()); - saveFile(newConfigBais, configFile); - final String confDir = bootstrapProperties.getProperty(CONF_DIR_KEY); - - try { - // Reset the input stream to provide to the transformer - newConfigBais.reset(); - - logger.info("Performing transformation for input and saving outputs to {}", confDir); - ByteBuffer tempConfigFile = runner.performTransformation(newConfigBais, confDir); - runner.currentConfigFileReference.set(tempConfigFile.asReadOnlyBuffer()); - - try { - logger.info("Reloading instance with new configuration"); - restartInstance(); - } catch (Exception e) { - logger.debug("Transformation of new config file failed after transformation into Flow.xml and nifi.properties, reverting."); - ByteBuffer resetConfigFile = runner.performTransformation(new FileInputStream(swapConfigFile), confDir); - runner.currentConfigFileReference.set(resetConfigFile.asReadOnlyBuffer()); - throw e; - } - } catch (Exception e) { - logger.debug("Transformation of new config file failed after replacing original with the swap file, reverting."); - Files.copy(new FileInputStream(swapConfigFile), configFile.toPath(), REPLACE_EXISTING); - throw e; - } - } catch (Exception e) { - logger.debug("Transformation of new config file failed after swap file was created, deleting it."); - if (!swapConfigFile.delete()) { - logger.warn("The swap file failed to delete after a failed handling of a change. It should be cleaned up manually."); - } - throw e; - } - } - } catch (ConfigurationChangeException e){ - logger.error("Unable to carry out reloading of configuration on receipt of notification event", e); - throw e; - } catch (IOException ioe) { - logger.error("Unable to carry out reloading of configuration on receipt of notification event", ioe); - throw new ConfigurationChangeException("Unable to perform reload of received configuration change", ioe); - } finally { - try { - if (configInputStream != null) { - configInputStream.close() ; - } - } catch (IOException e) { - // Quietly close - } - handlingLock.unlock(); - } - } - - @Override - public String getDescriptor() { - return "MiNiFiConfigurationChangeListener"; - } - - private void saveFile(final InputStream configInputStream, File configFile) throws IOException { - try { - try (final FileOutputStream configFileOutputStream = new FileOutputStream(configFile)) { - byte[] copyArray = new byte[1024]; - int available = -1; - while ((available = configInputStream.read(copyArray)) > 0) { - configFileOutputStream.write(copyArray, 0, available); - } - } - } catch (IOException ioe) { - throw new IOException("Unable to save updated configuration to the configured config file location", ioe); - } - } - - private void restartInstance() throws IOException { - try { - runner.reload(); - } catch (IOException e) { - throw new IOException("Unable to successfully restart MiNiFi instance after configuration change.", e); - } - } + @Override + public AtomicReference getConfigFileReference() { + return currentConfigFileReference; } - private ByteBuffer performTransformation(InputStream configIs, String configDestinationPath) throws ConfigurationChangeException, IOException { - try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); - TeeInputStream teeInputStream = new TeeInputStream(configIs, byteArrayOutputStream)) { - - ConfigTransformer.transformConfigFile( - teeInputStream, - configDestinationPath, - getBootstrapProperties() - ); - - return ByteBuffer.wrap(byteArrayOutputStream.toByteArray()); - } catch (ConfigurationChangeException e){ - throw e; - } catch (Exception e) { - throw new IOException("Unable to successfully transform the provided configuration", e); - } - } - - private static class Status { - - private final Integer port; - private final String pid; - - private final Boolean respondingToPing; - private final Boolean processRunning; - - public Status(final Integer port, final String pid, final Boolean respondingToPing, final Boolean processRunning) { - this.port = port; - this.pid = pid; - this.respondingToPing = respondingToPing; - this.processRunning = processRunning; - } - - public String getPid() { - return pid; - } - - public Integer getPort() { - return port; - } - - public boolean isRespondingToPing() { - return Boolean.TRUE.equals(respondingToPing); - } - - public boolean isProcessRunning() { - return Boolean.TRUE.equals(processRunning); - } + private ObjectMapper getObjectMapper() { + ObjectMapper objectMapper = new ObjectMapper(); + objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + objectMapper.setSerializationInclusion(JsonInclude.Include.NON_NULL); + objectMapper.enable(DeserializationFeature.READ_ENUMS_USING_TO_STRING); + return objectMapper; } } diff --git a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/SensitiveProperty.java b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/SensitiveProperty.java new file mode 100644 index 0000000000..84c7fbf473 --- /dev/null +++ b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/SensitiveProperty.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.minifi.bootstrap; + +import static java.util.stream.Collectors.collectingAndThen; +import static java.util.stream.Collectors.toSet; + +import java.util.Arrays; +import java.util.Collections; +import java.util.Set; + +public enum SensitiveProperty { + SECRET_KEY("secret.key"), + C2_SECURITY_TRUSTSTORE_PASSWORD("c2.security.truststore.password"), + C2_SECURITY_KEYSTORE_PASSWORD("c2.security.keystore.password"), + NIFI_MINIFI_SECURITY_KEYSTORE_PASSWORD("nifi.minifi.security.keystorePasswd"), + NIFI_MINIFI_SECURITY_TRUSTSTORE_PASSWORD("nifi.minifi.security.truststorePasswd"), + NIFI_MINIFI_SENSITIVE_PROPS_KEY("nifi.minifi.sensitive.props.key"); + + public static final Set SENSITIVE_PROPERTIES = Arrays.stream(SensitiveProperty.values()).map(SensitiveProperty::getKey) + .collect(collectingAndThen(toSet(), Collections::unmodifiableSet)); + + private final String key; + + SensitiveProperty(String key) { + this.key = key; + } + + public String getKey() { + return key; + } +} diff --git a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/ShutdownHook.java b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/ShutdownHook.java index 3aabc96f7c..5126475e27 100644 --- a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/ShutdownHook.java +++ b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/ShutdownHook.java @@ -17,107 +17,30 @@ package org.apache.nifi.minifi.bootstrap; -import java.io.File; -import java.io.IOException; -import java.io.OutputStream; -import java.net.Socket; -import java.nio.charset.StandardCharsets; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.TimeUnit; - -import org.apache.nifi.minifi.bootstrap.status.PeriodicStatusReporter; -import org.apache.nifi.minifi.bootstrap.configuration.ConfigurationChangeCoordinator; +import org.apache.nifi.minifi.bootstrap.service.MiNiFiStdLogHandler; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class ShutdownHook extends Thread { - private final Process nifiProcess; + private static final Logger LOGGER = LoggerFactory.getLogger("org.apache.nifi.minifi.bootstrap.Command"); + private final RunMiNiFi runner; - private final int gracefulShutdownSeconds; - private final ExecutorService executor; + private final MiNiFiStdLogHandler miNiFiStdLogHandler; - private volatile String secretKey; - - public ShutdownHook(final Process nifiProcess, final RunMiNiFi runner, final String secretKey, final int gracefulShutdownSeconds, final ExecutorService executor) { - this.nifiProcess = nifiProcess; + public ShutdownHook(RunMiNiFi runner, MiNiFiStdLogHandler miNiFiStdLogHandler) { this.runner = runner; - this.secretKey = secretKey; - this.gracefulShutdownSeconds = gracefulShutdownSeconds; - this.executor = executor; - } - - void setSecretKey(final String secretKey) { - this.secretKey = secretKey; + this.miNiFiStdLogHandler = miNiFiStdLogHandler; } @Override public void run() { - executor.shutdown(); - - System.out.println("Initiating shutdown of bootstrap change ingestors..."); - ConfigurationChangeCoordinator notifier = runner.getChangeCoordinator(); - if (notifier != null) { - try { - notifier.close(); - } catch (IOException ioe) { - System.out.println("Could not successfully stop notifier due to " + ioe); - } - } - - System.out.println("Initiating shutdown of bootstrap periodic status reporters..."); - for (PeriodicStatusReporter periodicStatusReporter : runner.getPeriodicStatusReporters()) { - try { - periodicStatusReporter.stop(); - } catch (Exception exception) { - System.out.println("Could not successfully stop periodic status reporter " + periodicStatusReporter.getClass() + " due to " + exception); - } - } + LOGGER.info("Initiating Shutdown of MiNiFi..."); + miNiFiStdLogHandler.shutdown(); + runner.shutdownChangeNotifier(); + runner.getPeriodicStatusReporterManager().shutdownPeriodicStatusReporters(); runner.setAutoRestartNiFi(false); - final int ccPort = runner.getNiFiCommandControlPort(); - if (ccPort > 0) { - System.out.println("Initiating Shutdown of MiNiFi..."); - - try { - final Socket socket = new Socket("localhost", ccPort); - final OutputStream out = socket.getOutputStream(); - out.write(("SHUTDOWN " + secretKey + "\n").getBytes(StandardCharsets.UTF_8)); - out.flush(); - - socket.close(); - } catch (final IOException ioe) { - System.out.println("Failed to Shutdown MiNiFi due to " + ioe); - } - } - - - System.out.println("Waiting for Apache MiNiFi to finish shutting down..."); - final long startWait = System.nanoTime(); - while (RunMiNiFi.isAlive(nifiProcess)) { - final long waitNanos = System.nanoTime() - startWait; - final long waitSeconds = TimeUnit.NANOSECONDS.toSeconds(waitNanos); - if (waitSeconds >= gracefulShutdownSeconds && gracefulShutdownSeconds > 0) { - if (RunMiNiFi.isAlive(nifiProcess)) { - System.out.println("MiNiFi has not finished shutting down after " + gracefulShutdownSeconds + " seconds. Killing process."); - nifiProcess.destroy(); - } - break; - } else { - try { - Thread.sleep(1000L); - } catch (final InterruptedException ie) { - } - } - } - - try { - final File statusFile = runner.getStatusFile(); - if (!statusFile.delete()) { - System.err.println("Failed to delete status file " + statusFile.getAbsolutePath() + "; this file should be cleaned up manually"); - } - }catch (IOException ex){ - System.err.println("Failed to retrieve status file " + ex); - } - - System.out.println("MiNiFi is done shutting down"); + runner.run(BootstrapCommand.STOP); } } diff --git a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/differentiators/interfaces/Differentiator.java b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/Status.java similarity index 67% rename from minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/differentiators/interfaces/Differentiator.java rename to minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/Status.java index 5beb78ba05..5dec38e0cb 100644 --- a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/differentiators/interfaces/Differentiator.java +++ b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/Status.java @@ -15,15 +15,21 @@ * limitations under the License. */ -package org.apache.nifi.minifi.bootstrap.configuration.differentiators.interfaces; +package org.apache.nifi.minifi.bootstrap; -import org.apache.nifi.minifi.bootstrap.ConfigurationFileHolder; +public enum Status { + OK(0), + ERROR(1), + MINIFI_NOT_RESPONDING(4), + MINIFI_NOT_RUNNING(3); -import java.io.IOException; -import java.util.Properties; + private final int statusCode; -public interface Differentiator { - void initialize(Properties properties, ConfigurationFileHolder configurationFileHolder); + Status(int statusCode) { + this.statusCode = statusCode; + } - boolean isNew(T input) throws IOException; + public int getStatusCode() { + return statusCode; + } } diff --git a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/WindowsService.java b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/WindowsService.java index 137e610420..b6dedf7fc2 100644 --- a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/WindowsService.java +++ b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/WindowsService.java @@ -17,26 +17,20 @@ package org.apache.nifi.minifi.bootstrap; import java.io.IOException; -import java.io.File; +import org.apache.nifi.minifi.bootstrap.service.BootstrapFileProvider; public class WindowsService { private static RunMiNiFi bootstrap; - public static void start(String[] args) throws IOException, InterruptedException { - - final File bootstrapConfigFile = RunMiNiFi.getBootstrapConfFile(); - - bootstrap = new RunMiNiFi(bootstrapConfigFile); - bootstrap.start(); - + public static void start(String[] args) throws IOException { + bootstrap = new RunMiNiFi(BootstrapFileProvider.getBootstrapConfFile()); + bootstrap.run(BootstrapCommand.START); } - public static void stop(String[] args) throws IOException, InterruptedException { - + public static void stop(String[] args) { bootstrap.setAutoRestartNiFi(false); - bootstrap.stop(); - + bootstrap.run(BootstrapCommand.STOP); } } diff --git a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/command/CommandRunner.java b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/command/CommandRunner.java new file mode 100644 index 0000000000..536f17b592 --- /dev/null +++ b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/command/CommandRunner.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.minifi.bootstrap.command; + +public interface CommandRunner { + + /** + * Executes a command. + * @param args the input arguments + * @return status code + */ + int runCommand(String[] args); +} diff --git a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/command/CommandRunnerFactory.java b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/command/CommandRunnerFactory.java new file mode 100644 index 0000000000..6b11d5836a --- /dev/null +++ b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/command/CommandRunnerFactory.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.minifi.bootstrap.command; + +import java.io.File; +import java.util.LinkedList; +import java.util.List; +import org.apache.nifi.minifi.bootstrap.BootstrapCommand; +import org.apache.nifi.minifi.bootstrap.MiNiFiParameters; +import org.apache.nifi.minifi.bootstrap.RunMiNiFi; +import org.apache.nifi.minifi.bootstrap.service.BootstrapFileProvider; +import org.apache.nifi.minifi.bootstrap.service.CurrentPortProvider; +import org.apache.nifi.minifi.bootstrap.service.GracefulShutdownParameterProvider; +import org.apache.nifi.minifi.bootstrap.service.MiNiFiCommandSender; +import org.apache.nifi.minifi.bootstrap.service.MiNiFiExecCommandProvider; +import org.apache.nifi.minifi.bootstrap.service.MiNiFiStatusProvider; +import org.apache.nifi.minifi.bootstrap.service.MiNiFiStdLogHandler; +import org.apache.nifi.minifi.bootstrap.service.PeriodicStatusReporterManager; + +public class CommandRunnerFactory { + + private final MiNiFiCommandSender miNiFiCommandSender; + private final CurrentPortProvider currentPortProvider; + private final MiNiFiParameters miNiFiParameters; + private final MiNiFiStatusProvider miNiFiStatusProvider; + private final PeriodicStatusReporterManager periodicStatusReporterManager; + private final BootstrapFileProvider bootstrapFileProvider; + private final MiNiFiStdLogHandler miNiFiStdLogHandler; + private final File bootstrapConfigFile; + private final RunMiNiFi runMiNiFi; + private final GracefulShutdownParameterProvider gracefulShutdownParameterProvider; + private final MiNiFiExecCommandProvider miNiFiExecCommandProvider; + + public CommandRunnerFactory(MiNiFiCommandSender miNiFiCommandSender, CurrentPortProvider currentPortProvider, MiNiFiParameters miNiFiParameters, + MiNiFiStatusProvider miNiFiStatusProvider, PeriodicStatusReporterManager periodicStatusReporterManager, + BootstrapFileProvider bootstrapFileProvider, MiNiFiStdLogHandler miNiFiStdLogHandler, File bootstrapConfigFile, RunMiNiFi runMiNiFi, + GracefulShutdownParameterProvider gracefulShutdownParameterProvider, MiNiFiExecCommandProvider miNiFiExecCommandProvider) { + this.miNiFiCommandSender = miNiFiCommandSender; + this.currentPortProvider = currentPortProvider; + this.miNiFiParameters = miNiFiParameters; + this.miNiFiStatusProvider = miNiFiStatusProvider; + this.periodicStatusReporterManager = periodicStatusReporterManager; + this.bootstrapFileProvider = bootstrapFileProvider; + this.miNiFiStdLogHandler = miNiFiStdLogHandler; + this.bootstrapConfigFile = bootstrapConfigFile; + this.runMiNiFi = runMiNiFi; + this.gracefulShutdownParameterProvider = gracefulShutdownParameterProvider; + this.miNiFiExecCommandProvider = miNiFiExecCommandProvider; + } + + /** + * Returns a runner associated with the given command. + * @param command the bootstrap command + * @return the runner + */ + public CommandRunner getRunner(BootstrapCommand command) { + CommandRunner commandRunner; + switch (command) { + case START: + case RUN: + commandRunner = new StartRunner(currentPortProvider, bootstrapFileProvider, periodicStatusReporterManager, miNiFiStdLogHandler, miNiFiParameters, + bootstrapConfigFile, runMiNiFi, miNiFiExecCommandProvider); + break; + case STOP: + commandRunner = new StopRunner(bootstrapFileProvider, miNiFiParameters, miNiFiCommandSender, currentPortProvider, gracefulShutdownParameterProvider); + break; + case STATUS: + commandRunner = new StatusRunner(miNiFiParameters, miNiFiStatusProvider); + break; + case RESTART: + commandRunner = new CompositeCommandRunner(getRestartServices()); + break; + case DUMP: + commandRunner = new DumpRunner(miNiFiCommandSender, currentPortProvider); + break; + case ENV: + commandRunner = new EnvRunner(miNiFiCommandSender, currentPortProvider); + break; + case FLOWSTATUS: + commandRunner = new FlowStatusRunner(periodicStatusReporterManager); + break; + default: + throw new IllegalArgumentException("Unknown MiNiFi bootstrap command"); + } + return commandRunner; + } + + private List getRestartServices() { + List compositeList = new LinkedList<>(); + compositeList.add(new StopRunner(bootstrapFileProvider, miNiFiParameters, miNiFiCommandSender, currentPortProvider, gracefulShutdownParameterProvider)); + compositeList.add(new StartRunner(currentPortProvider, bootstrapFileProvider, periodicStatusReporterManager, miNiFiStdLogHandler, miNiFiParameters, + bootstrapConfigFile, runMiNiFi, miNiFiExecCommandProvider)); + return compositeList; + } +} diff --git a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/command/CompositeCommandRunner.java b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/command/CompositeCommandRunner.java new file mode 100644 index 0000000000..484df20299 --- /dev/null +++ b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/command/CompositeCommandRunner.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.minifi.bootstrap.command; + +import static org.apache.nifi.minifi.bootstrap.Status.OK; + +import java.util.Collections; +import java.util.List; +import java.util.Optional; + +/** + * Composite runner which can execute multiple commands in a sequential order. + */ +public class CompositeCommandRunner implements CommandRunner { + final List services; + + public CompositeCommandRunner(List services) { + this.services = Optional.ofNullable(services).map(Collections::unmodifiableList).orElse(Collections.emptyList()); + } + + /** + * Executes the runners in sequential order. Stops on first failure. + * @param args the input arguments + * @return the first failed command status code or OK if there was no failure + */ + @Override + public int runCommand(String[] args) { + return services.stream() + .map(service -> service.runCommand(args)) + .filter(code -> code != OK.getStatusCode()) + .findFirst() + .orElse(OK.getStatusCode()); + } +} diff --git a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/command/DumpRunner.java b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/command/DumpRunner.java new file mode 100644 index 0000000000..242480b019 --- /dev/null +++ b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/command/DumpRunner.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.minifi.bootstrap.command; + +import static org.apache.nifi.minifi.bootstrap.RunMiNiFi.CMD_LOGGER; +import static org.apache.nifi.minifi.bootstrap.RunMiNiFi.DEFAULT_LOGGER; +import static org.apache.nifi.minifi.bootstrap.Status.ERROR; +import static org.apache.nifi.minifi.bootstrap.Status.MINIFI_NOT_RUNNING; +import static org.apache.nifi.minifi.bootstrap.Status.OK; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Optional; +import org.apache.nifi.minifi.bootstrap.service.CurrentPortProvider; +import org.apache.nifi.minifi.bootstrap.service.MiNiFiCommandSender; + +public class DumpRunner implements CommandRunner { + private static final String DUMP_CMD = "DUMP"; + + private final MiNiFiCommandSender miNiFiCommandSender; + private final CurrentPortProvider currentPortProvider; + + public DumpRunner(MiNiFiCommandSender miNiFiCommandSender, CurrentPortProvider currentPortProvider) { + this.miNiFiCommandSender = miNiFiCommandSender; + this.currentPortProvider = currentPortProvider; + } + + /** + * Writes a MiNiFi thread dump to the given file; if file is null, logs at + * INFO level instead. + * + * @param args the second parameter is the file to write the dump content to + */ + @Override + public int runCommand(String[] args) { + return dump(getArg(args, 1).map(File::new).orElse(null)); + } + + private int dump(File dumpFile) { + Integer port = currentPortProvider.getCurrentPort(); + if (port == null) { + CMD_LOGGER.error("Apache MiNiFi is not currently running"); + return MINIFI_NOT_RUNNING.getStatusCode(); + } + + Optional dump; + try { + dump = miNiFiCommandSender.sendCommand(DUMP_CMD, port); + } catch (IOException e) { + CMD_LOGGER.error("Failed to get DUMP response from MiNiFi"); + DEFAULT_LOGGER.error("Exception:", e); + return ERROR.getStatusCode(); + } + + return Optional.ofNullable(dumpFile) + .map(dmp -> writeDumpToFile(dmp, dump)) + .orElseGet(() -> { + dump.ifPresent(CMD_LOGGER::info); + return OK.getStatusCode(); + }); + } + + private Integer writeDumpToFile(File dumpFile, Optional dump) { + try (FileOutputStream fos = new FileOutputStream(dumpFile)) { + fos.write(dump.orElse("Dump has empty response").getBytes(StandardCharsets.UTF_8)); + } catch (IOException e) { + CMD_LOGGER.error("Failed to write DUMP response to file"); + DEFAULT_LOGGER.error("Exception:", e); + return ERROR.getStatusCode(); + } + // we want to log to the console (by default) that we wrote the thread dump to the specified file + CMD_LOGGER.info("Successfully wrote thread dump to {}", dumpFile.getAbsolutePath()); + return OK.getStatusCode(); + } + + private Optional getArg(String[] args, int index) { + return Optional.ofNullable(args.length > index ? args[index] : null); + } +} diff --git a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/command/EnvRunner.java b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/command/EnvRunner.java new file mode 100644 index 0000000000..2a0cfc004b --- /dev/null +++ b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/command/EnvRunner.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.minifi.bootstrap.command; + +import static org.apache.nifi.minifi.bootstrap.RunMiNiFi.CMD_LOGGER; +import static org.apache.nifi.minifi.bootstrap.RunMiNiFi.DEFAULT_LOGGER; +import static org.apache.nifi.minifi.bootstrap.Status.ERROR; +import static org.apache.nifi.minifi.bootstrap.Status.MINIFI_NOT_RUNNING; +import static org.apache.nifi.minifi.bootstrap.Status.OK; + +import java.io.IOException; +import org.apache.nifi.minifi.bootstrap.service.CurrentPortProvider; +import org.apache.nifi.minifi.bootstrap.service.MiNiFiCommandSender; + +public class EnvRunner implements CommandRunner { + private static final String ENV_CMD = "ENV"; + + private final MiNiFiCommandSender miNiFiCommandSender; + private final CurrentPortProvider currentPortProvider; + + public EnvRunner(MiNiFiCommandSender miNiFiCommandSender, CurrentPortProvider currentPortProvider) { + this.miNiFiCommandSender = miNiFiCommandSender; + this.currentPortProvider = currentPortProvider; + } + + /** + * Returns information about the MiNiFi's virtual machine. + * @param args the input arguments + * @return status code + */ + @Override + public int runCommand(String[] args) { + return env(); + } + + private int env() { + Integer port = currentPortProvider.getCurrentPort(); + if (port == null) { + CMD_LOGGER.error("Apache MiNiFi is not currently running"); + return MINIFI_NOT_RUNNING.getStatusCode(); + } + + try { + miNiFiCommandSender.sendCommand(ENV_CMD, port).ifPresent(CMD_LOGGER::info); + } catch (IOException e) { + CMD_LOGGER.error("Failed to get ENV response from MiNiFi"); + DEFAULT_LOGGER.error("Exception:", e); + return ERROR.getStatusCode(); + } + + return OK.getStatusCode(); + } +} diff --git a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/command/FlowStatusRunner.java b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/command/FlowStatusRunner.java new file mode 100644 index 0000000000..7b0ea43a7b --- /dev/null +++ b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/command/FlowStatusRunner.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.minifi.bootstrap.command; + +import static org.apache.nifi.minifi.bootstrap.RunMiNiFi.CMD_LOGGER; +import static org.apache.nifi.minifi.bootstrap.Status.ERROR; +import static org.apache.nifi.minifi.bootstrap.Status.OK; + +import org.apache.nifi.minifi.bootstrap.service.PeriodicStatusReporterManager; + +public class FlowStatusRunner implements CommandRunner { + private final PeriodicStatusReporterManager periodicStatusReporterManager; + + public FlowStatusRunner(PeriodicStatusReporterManager periodicStatusReporterManager) { + this.periodicStatusReporterManager = periodicStatusReporterManager; + } + + /** + * Receive and print detailed flow information from MiNiFi. + * Example query: processor:TailFile:health,stats,bulletins + * @param args the input arguments + * @return status code + */ + @Override + public int runCommand(String[] args) { + if(args.length == 2) { + CMD_LOGGER.info(periodicStatusReporterManager.statusReport(args[1]).toString()); + return OK.getStatusCode(); + } else { + CMD_LOGGER.error("The 'flowStatus' command requires an input query. See the System Admin Guide 'FlowStatus Script Query' section for complete details."); + return ERROR.getStatusCode(); + } + } +} diff --git a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/command/StartRunner.java b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/command/StartRunner.java new file mode 100644 index 0000000000..513760f5a8 --- /dev/null +++ b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/command/StartRunner.java @@ -0,0 +1,329 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.minifi.bootstrap.command; + +import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; +import static org.apache.nifi.minifi.bootstrap.RunMiNiFi.CMD_LOGGER; +import static org.apache.nifi.minifi.bootstrap.RunMiNiFi.CONF_DIR_KEY; +import static org.apache.nifi.minifi.bootstrap.RunMiNiFi.DEFAULT_LOGGER; +import static org.apache.nifi.minifi.bootstrap.RunMiNiFi.MINIFI_CONFIG_FILE_KEY; +import static org.apache.nifi.minifi.bootstrap.RunMiNiFi.STATUS_FILE_PID_KEY; +import static org.apache.nifi.minifi.bootstrap.Status.ERROR; +import static org.apache.nifi.minifi.bootstrap.Status.OK; +import static org.apache.nifi.minifi.bootstrap.util.ConfigTransformer.generateConfigFiles; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.List; +import java.util.Optional; +import java.util.Properties; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.Condition; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; +import java.util.stream.Collectors; +import org.apache.nifi.bootstrap.util.OSUtils; +import org.apache.nifi.minifi.bootstrap.MiNiFiParameters; +import org.apache.nifi.minifi.bootstrap.RunMiNiFi; +import org.apache.nifi.minifi.bootstrap.ShutdownHook; +import org.apache.nifi.minifi.bootstrap.configuration.ConfigurationChangeException; +import org.apache.nifi.minifi.bootstrap.exception.StartupFailureException; +import org.apache.nifi.minifi.bootstrap.service.BootstrapFileProvider; +import org.apache.nifi.minifi.bootstrap.service.CurrentPortProvider; +import org.apache.nifi.minifi.bootstrap.service.MiNiFiExecCommandProvider; +import org.apache.nifi.minifi.bootstrap.service.MiNiFiListener; +import org.apache.nifi.minifi.bootstrap.service.MiNiFiStdLogHandler; +import org.apache.nifi.minifi.bootstrap.service.PeriodicStatusReporterManager; +import org.apache.nifi.minifi.bootstrap.util.UnixProcessUtils; +import org.apache.nifi.util.Tuple; + +public class StartRunner implements CommandRunner { + private static final int STARTUP_WAIT_SECONDS = 60; + + private final CurrentPortProvider currentPortProvider; + private final BootstrapFileProvider bootstrapFileProvider; + private final PeriodicStatusReporterManager periodicStatusReporterManager; + private final MiNiFiStdLogHandler miNiFiStdLogHandler; + private final MiNiFiParameters miNiFiParameters; + private final File bootstrapConfigFile; + private final Lock lock = new ReentrantLock(); + private final Condition startupCondition = lock.newCondition(); + private final RunMiNiFi runMiNiFi; + private volatile ShutdownHook shutdownHook; + private final MiNiFiExecCommandProvider miNiFiExecCommandProvider; + + public StartRunner(CurrentPortProvider currentPortProvider, BootstrapFileProvider bootstrapFileProvider, + PeriodicStatusReporterManager periodicStatusReporterManager, MiNiFiStdLogHandler miNiFiStdLogHandler, MiNiFiParameters miNiFiParameters, File bootstrapConfigFile, + RunMiNiFi runMiNiFi, MiNiFiExecCommandProvider miNiFiExecCommandProvider) { + this.currentPortProvider = currentPortProvider; + this.bootstrapFileProvider = bootstrapFileProvider; + this.periodicStatusReporterManager = periodicStatusReporterManager; + this.miNiFiStdLogHandler = miNiFiStdLogHandler; + this.miNiFiParameters = miNiFiParameters; + this.bootstrapConfigFile = bootstrapConfigFile; + this.runMiNiFi = runMiNiFi; + this.miNiFiExecCommandProvider = miNiFiExecCommandProvider; + } + + /** + * Starts (and restarts) MiNiFi process during the whole lifecycle of the bootstrap process. + * @param args the input arguments + * @return status code + */ + @Override + public int runCommand(String[] args) { + try { + start(); + } catch (Exception e) { + CMD_LOGGER.error("Exception happened during MiNiFi startup", e); + return ERROR.getStatusCode(); + } + return OK.getStatusCode(); + } + + private void start() throws IOException, InterruptedException { + Integer port = currentPortProvider.getCurrentPort(); + if (port != null) { + CMD_LOGGER.info("Apache MiNiFi is already running, listening to Bootstrap on port {}", port); + return; + } + + File prevLockFile = bootstrapFileProvider.getLockFile(); + if (prevLockFile.exists() && !prevLockFile.delete()) { + CMD_LOGGER.warn("Failed to delete previous lock file {}; this file should be cleaned up manually", prevLockFile); + } + + Properties bootstrapProperties = bootstrapFileProvider.getBootstrapProperties(); + String confDir = bootstrapProperties.getProperty(CONF_DIR_KEY); + initConfigFiles(bootstrapProperties, confDir); + + Tuple tuple = startMiNiFi(); + ProcessBuilder builder = tuple.getKey(); + Process process = tuple.getValue(); + + try { + while (true) { + if (UnixProcessUtils.isAlive(process)) { + handleReload(); + } else { + Runtime runtime = Runtime.getRuntime(); + try { + runtime.removeShutdownHook(shutdownHook); + } catch (IllegalStateException ise) { + DEFAULT_LOGGER.trace("The virtual machine is already in the process of shutting down", ise); + } + + if (runMiNiFi.isAutoRestartNiFi() && needRestart()) { + File reloadFile = bootstrapFileProvider.getReloadLockFile(); + if (reloadFile.exists()) { + DEFAULT_LOGGER.info("Currently reloading configuration. Will wait to restart MiNiFi."); + Thread.sleep(5000L); + continue; + } + + process = restartNifi(bootstrapProperties, confDir, builder, runtime); + // failed to start process + if (process == null) { + return; + } + } else { + return; + } + } + } + } finally { + miNiFiStdLogHandler.shutdown(); + runMiNiFi.shutdownChangeNotifier(); + periodicStatusReporterManager.shutdownPeriodicStatusReporters(); + } + } + + private Process restartNifi(Properties bootstrapProperties, String confDir, ProcessBuilder builder, Runtime runtime) throws IOException { + Process process; + boolean previouslyStarted = runMiNiFi.isNiFiStarted(); + if (!previouslyStarted) { + File swapConfigFile = bootstrapFileProvider.getSwapFile(); + if (swapConfigFile.exists()) { + DEFAULT_LOGGER.info("Swap file exists, MiNiFi failed trying to change configuration. Reverting to old configuration."); + + try { + ByteBuffer tempConfigFile = generateConfigFiles(new FileInputStream(swapConfigFile), confDir, bootstrapProperties); + runMiNiFi.getConfigFileReference().set(tempConfigFile.asReadOnlyBuffer()); + } catch (ConfigurationChangeException e) { + DEFAULT_LOGGER.error("The swap file is malformed, unable to restart from prior state. Will not attempt to restart MiNiFi. Swap File should be cleaned up manually."); + return null; + } + + Files.copy(swapConfigFile.toPath(), Paths.get(bootstrapProperties.getProperty(MINIFI_CONFIG_FILE_KEY)), REPLACE_EXISTING); + + DEFAULT_LOGGER.info("Replacing config file with swap file and deleting swap file"); + if (!swapConfigFile.delete()) { + DEFAULT_LOGGER.warn("The swap file failed to delete after replacing using it to revert to the old configuration. It should be cleaned up manually."); + } + runMiNiFi.setReloading(false); + } else { + DEFAULT_LOGGER.info("MiNiFi either never started or failed to restart. Will not attempt to restart MiNiFi"); + return null; + } + } else { + runMiNiFi.setNiFiStarted(false); + } + + miNiFiParameters.setSecretKey(null); + + process = startMiNiFiProcess(builder); + + boolean started = waitForStart(); + + if (started) { + Long pid = OSUtils.getProcessId(process, DEFAULT_LOGGER); + DEFAULT_LOGGER.info("Successfully spawned the thread to start Apache MiNiFi{}", (pid == null ? "" : " with PID " + pid)); + } else { + DEFAULT_LOGGER.error("Apache MiNiFi does not appear to have started"); + } + return process; + } + + private boolean needRestart() throws IOException { + boolean needRestart = true; + File statusFile = bootstrapFileProvider.getStatusFile(); + if (!statusFile.exists()) { + DEFAULT_LOGGER.info("Status File no longer exists. Will not restart MiNiFi"); + return false; + } + + File lockFile = bootstrapFileProvider.getLockFile(); + if (lockFile.exists()) { + DEFAULT_LOGGER.info("A shutdown was initiated. Will not restart MiNiFi"); + return false; + } + return needRestart; + } + + private void handleReload() { + try { + Thread.sleep(1000L); + if (runMiNiFi.getReloading() && runMiNiFi.isNiFiStarted()) { + File swapConfigFile = bootstrapFileProvider.getSwapFile(); + if (swapConfigFile.exists()) { + DEFAULT_LOGGER.info("MiNiFi has finished reloading successfully and swap file exists. Deleting old configuration."); + + if (swapConfigFile.delete()) { + DEFAULT_LOGGER.info("Swap file was successfully deleted."); + } else { + DEFAULT_LOGGER.error("Swap file was not deleted. It should be deleted manually."); + } + } + runMiNiFi.setReloading(false); + } + } catch (InterruptedException ie) { + } + } + + private void initConfigFiles(Properties bootstrapProperties, String confDir) throws IOException { + File configFile = new File(bootstrapProperties.getProperty(MINIFI_CONFIG_FILE_KEY)); + try (InputStream inputStream = new FileInputStream(configFile)) { + ByteBuffer tempConfigFile = generateConfigFiles(inputStream, confDir, bootstrapProperties); + runMiNiFi.getConfigFileReference().set(tempConfigFile.asReadOnlyBuffer()); + } catch (FileNotFoundException e) { + String fileNotFoundMessage = "The config file defined in " + MINIFI_CONFIG_FILE_KEY + " does not exists."; + DEFAULT_LOGGER.error(fileNotFoundMessage, e); + throw new StartupFailureException(fileNotFoundMessage); + } catch (ConfigurationChangeException e) { + String malformedConfigFileMessage = "The config file is malformed, unable to start."; + DEFAULT_LOGGER.error(malformedConfigFileMessage, e); + throw new StartupFailureException(malformedConfigFileMessage); + } + } + + private Tuple startMiNiFi() throws IOException { + ProcessBuilder builder = new ProcessBuilder(); + + File workingDir = getWorkingDir(); + MiNiFiListener listener = new MiNiFiListener(); + int listenPort = listener.start(runMiNiFi); + List cmd = miNiFiExecCommandProvider.getMiNiFiExecCommand(listenPort, workingDir); + + builder.command(cmd); + builder.directory(workingDir); + + CMD_LOGGER.info("Starting Apache MiNiFi..."); + CMD_LOGGER.info("Working Directory: {}", workingDir.getAbsolutePath()); + CMD_LOGGER.info("Command: {}", cmd.stream().collect(Collectors.joining(" "))); + + return new Tuple<>(builder, startMiNiFiProcess(builder)); + } + + private Process startMiNiFiProcess(ProcessBuilder builder) throws IOException { + Process process = builder.start(); + miNiFiStdLogHandler.initLogging(process); + Long pid = OSUtils.getProcessId(process, CMD_LOGGER); + if (pid != null) { + miNiFiParameters.setMinifiPid(pid); + Properties minifiProps = new Properties(); + minifiProps.setProperty(STATUS_FILE_PID_KEY, String.valueOf(pid)); + bootstrapFileProvider.saveStatusProperties(minifiProps); + } + + shutdownHook = new ShutdownHook(runMiNiFi, miNiFiStdLogHandler); + Runtime.getRuntime().addShutdownHook(shutdownHook); + return process; + } + + private File getWorkingDir() throws IOException { + Properties props = bootstrapFileProvider.getBootstrapProperties(); + File bootstrapConfigAbsoluteFile = bootstrapConfigFile.getAbsoluteFile(); + File binDir = bootstrapConfigAbsoluteFile.getParentFile(); + + File workingDir = Optional.ofNullable(props.getProperty("working.dir")) + .map(File::new) + .orElse(binDir.getParentFile()); + return workingDir; + } + + private boolean waitForStart() { + lock.lock(); + try { + long startTime = System.nanoTime(); + + while (miNiFiParameters.getMinifiPid() < 1 && miNiFiParameters.getMiNiFiPort() < 1) { + try { + startupCondition.await(1, TimeUnit.SECONDS); + } catch (InterruptedException ie) { + return false; + } + + long waitNanos = System.nanoTime() - startTime; + long waitSeconds = TimeUnit.NANOSECONDS.toSeconds(waitNanos); + if (waitSeconds > STARTUP_WAIT_SECONDS) { + return false; + } + } + } finally { + lock.unlock(); + } + return true; + } + +} diff --git a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/command/StatusRunner.java b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/command/StatusRunner.java new file mode 100644 index 0000000000..c7e6e47dfa --- /dev/null +++ b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/command/StatusRunner.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.minifi.bootstrap.command; + +import static org.apache.nifi.minifi.bootstrap.RunMiNiFi.CMD_LOGGER; +import static org.apache.nifi.minifi.bootstrap.Status.MINIFI_NOT_RESPONDING; +import static org.apache.nifi.minifi.bootstrap.Status.MINIFI_NOT_RUNNING; +import static org.apache.nifi.minifi.bootstrap.Status.OK; + +import org.apache.nifi.minifi.bootstrap.MiNiFiParameters; +import org.apache.nifi.minifi.bootstrap.MiNiFiStatus; +import org.apache.nifi.minifi.bootstrap.service.MiNiFiStatusProvider; + +public class StatusRunner implements CommandRunner { + private final MiNiFiParameters miNiFiParameters; + private final MiNiFiStatusProvider miNiFiStatusProvider; + + public StatusRunner(MiNiFiParameters miNiFiParameters, MiNiFiStatusProvider miNiFiStatusProvider) { + this.miNiFiParameters = miNiFiParameters; + this.miNiFiStatusProvider = miNiFiStatusProvider; + } + + /** + * Prints the current status of the MiNiFi process. + * @param args the input arguments + * @return status code + */ + @Override + public int runCommand(String[] args) { + return status(); + } + + private int status() { + MiNiFiStatus status = miNiFiStatusProvider.getStatus(miNiFiParameters.getMiNiFiPort(), miNiFiParameters.getMinifiPid()); + if (status.isRespondingToPing()) { + CMD_LOGGER.info("Apache MiNiFi is currently running, listening to Bootstrap on port {}, PID={}", + status.getPort(), status.getPid() == null ? "unknown" : status.getPid()); + return OK.getStatusCode(); + } + + if (status.isProcessRunning()) { + CMD_LOGGER.info("Apache MiNiFi is running at PID {} but is not responding to ping requests", status.getPid()); + return MINIFI_NOT_RESPONDING.getStatusCode(); + } + + if (status.getPort() == null) { + CMD_LOGGER.info("Apache MiNiFi is not running"); + return MINIFI_NOT_RUNNING.getStatusCode(); + } + + if (status.getPid() == null) { + CMD_LOGGER.info("Apache MiNiFi is not responding to Ping requests. The process may have died or may be hung"); + } else { + CMD_LOGGER.info("Apache MiNiFi is not running"); + } + return MINIFI_NOT_RUNNING.getStatusCode(); + } +} diff --git a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/command/StopRunner.java b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/command/StopRunner.java new file mode 100644 index 0000000000..c712765a5e --- /dev/null +++ b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/command/StopRunner.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.minifi.bootstrap.command; + +import static org.apache.nifi.minifi.bootstrap.RunMiNiFi.CMD_LOGGER; +import static org.apache.nifi.minifi.bootstrap.RunMiNiFi.DEFAULT_LOGGER; +import static org.apache.nifi.minifi.bootstrap.RunMiNiFi.UNINITIALIZED; +import static org.apache.nifi.minifi.bootstrap.Status.ERROR; +import static org.apache.nifi.minifi.bootstrap.Status.MINIFI_NOT_RUNNING; +import static org.apache.nifi.minifi.bootstrap.Status.OK; + +import java.io.File; +import java.io.IOException; +import java.util.Optional; +import org.apache.nifi.minifi.bootstrap.MiNiFiParameters; +import org.apache.nifi.minifi.bootstrap.service.BootstrapFileProvider; +import org.apache.nifi.minifi.bootstrap.service.CurrentPortProvider; +import org.apache.nifi.minifi.bootstrap.service.GracefulShutdownParameterProvider; +import org.apache.nifi.minifi.bootstrap.service.MiNiFiCommandSender; +import org.apache.nifi.minifi.bootstrap.util.UnixProcessUtils; + +public class StopRunner implements CommandRunner { + private static final String SHUTDOWN_CMD = "SHUTDOWN"; + + private final BootstrapFileProvider bootstrapFileProvider; + private final MiNiFiParameters miNiFiParameters; + private final MiNiFiCommandSender miNiFiCommandSender; + private final CurrentPortProvider currentPortProvider; + private final GracefulShutdownParameterProvider gracefulShutdownParameterProvider; + + public StopRunner(BootstrapFileProvider bootstrapFileProvider, MiNiFiParameters miNiFiParameters, MiNiFiCommandSender miNiFiCommandSender, + CurrentPortProvider currentPortProvider, GracefulShutdownParameterProvider gracefulShutdownParameterProvider) { + this.bootstrapFileProvider = bootstrapFileProvider; + this.miNiFiParameters = miNiFiParameters; + this.miNiFiCommandSender = miNiFiCommandSender; + this.currentPortProvider = currentPortProvider; + this.gracefulShutdownParameterProvider = gracefulShutdownParameterProvider; + } + + /** + * Shutdown the MiNiFi and the managing bootstrap process as well. + * @param args the input arguments + * @return status code + */ + @Override + public int runCommand(String[] args) { + try { + return stop(); + } catch (Exception e) { + DEFAULT_LOGGER.error("Exception happened during stopping MiNiFi", e); + return ERROR.getStatusCode(); + } + } + + private int stop() throws IOException { + Integer currentPort = currentPortProvider.getCurrentPort(); + if (currentPort == null) { + CMD_LOGGER.error("Apache MiNiFi is not currently running"); + return MINIFI_NOT_RUNNING.getStatusCode(); + } + + int status = OK.getStatusCode(); + // indicate that a stop command is in progress + File lockFile = bootstrapFileProvider.getLockFile(); + if (!lockFile.exists()) { + lockFile.createNewFile(); + } + + File statusFile = bootstrapFileProvider.getStatusFile(); + File pidFile = bootstrapFileProvider.getPidFile(); + long minifiPid = miNiFiParameters.getMinifiPid(); + + try { + Optional commandResponse = miNiFiCommandSender.sendCommand(SHUTDOWN_CMD, currentPort); + if (commandResponse.filter(SHUTDOWN_CMD::equals).isPresent()) { + CMD_LOGGER.info("Apache MiNiFi has accepted the Shutdown Command and is shutting down now"); + + if (minifiPid != UNINITIALIZED) { + UnixProcessUtils.gracefulShutDownMiNiFiProcess(minifiPid, "MiNiFi has not finished shutting down after {} seconds. Killing process.", + gracefulShutdownParameterProvider.getGracefulShutdownSeconds()); + + if (statusFile.exists() && !statusFile.delete()) { + CMD_LOGGER.error("Failed to delete status file {}; this file should be cleaned up manually", statusFile); + } + + if (pidFile.exists() && !pidFile.delete()) { + CMD_LOGGER.error("Failed to delete pid file {}; this file should be cleaned up manually", pidFile); + } + + CMD_LOGGER.info("MiNiFi has finished shutting down."); + } + } else { + CMD_LOGGER.error("When sending SHUTDOWN command to MiNiFi, got unexpected response {}", commandResponse.orElse(null)); + status = ERROR.getStatusCode(); + } + } catch (IOException e) { + if (minifiPid == UNINITIALIZED) { + DEFAULT_LOGGER.error("No PID found for the MiNiFi process, so unable to kill process; The process should be killed manually."); + } else { + DEFAULT_LOGGER.error("Will kill the MiNiFi Process with PID {}", minifiPid); + UnixProcessUtils.killProcessTree(minifiPid); + } + } finally { + if (lockFile.exists() && !lockFile.delete()) { + CMD_LOGGER.error("Failed to delete lock file {}; this file should be cleaned up manually", lockFile); + } + } + + return status; + } +} diff --git a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/ConfigurationChangeCoordinator.java b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/ConfigurationChangeCoordinator.java index 3fa5b8fc4d..8bf255f07c 100644 --- a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/ConfigurationChangeCoordinator.java +++ b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/ConfigurationChangeCoordinator.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -16,61 +16,46 @@ */ package org.apache.nifi.minifi.bootstrap.configuration; -import org.apache.nifi.minifi.bootstrap.ConfigurationFileHolder; +import java.io.Closeable; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.Optional; +import java.util.Properties; +import java.util.Set; +import org.apache.nifi.minifi.bootstrap.RunMiNiFi; import org.apache.nifi.minifi.bootstrap.configuration.ingestors.interfaces.ChangeIngestor; import org.apache.nifi.minifi.bootstrap.util.ByteBufferInputStream; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.Closeable; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashSet; -import java.util.Properties; -import java.util.Set; - public class ConfigurationChangeCoordinator implements Closeable, ConfigurationChangeNotifier { public static final String NOTIFIER_PROPERTY_PREFIX = "nifi.minifi.notifier"; public static final String NOTIFIER_INGESTORS_KEY = NOTIFIER_PROPERTY_PREFIX + ".ingestors"; - private final static Logger logger = LoggerFactory.getLogger(ConfigurationChangeCoordinator.class); - private final Set configurationChangeListeners = new HashSet<>(); + private static final Logger LOGGER = LoggerFactory.getLogger(ConfigurationChangeCoordinator.class); + + private final Set configurationChangeListeners; private final Set changeIngestors = new HashSet<>(); - /** - * Provides an opportunity for the implementation to perform configuration and initialization based on properties received from the bootstrapping configuration - * - * @param properties from the bootstrap configuration - */ - public void initialize(Properties properties, ConfigurationFileHolder configurationFileHolder, Collection changeListenerSet) { - final String ingestorsCsv = properties.getProperty(NOTIFIER_INGESTORS_KEY); + private final Properties bootstrapProperties; + private final RunMiNiFi runMiNiFi; - if (ingestorsCsv != null && !ingestorsCsv.isEmpty()) { - for (String ingestorClassname : Arrays.asList(ingestorsCsv.split(","))) { - ingestorClassname = ingestorClassname.trim(); - try { - Class ingestorClass = Class.forName(ingestorClassname); - ChangeIngestor changeIngestor = (ChangeIngestor) ingestorClass.newInstance(); - changeIngestor.initialize(properties, configurationFileHolder, this); - changeIngestors.add(changeIngestor); - logger.info("Initialized "); - } catch (InstantiationException | IllegalAccessException | ClassNotFoundException e) { - throw new RuntimeException("Issue instantiating ingestor " + ingestorClassname, e); - } - } - } - configurationChangeListeners.clear(); - configurationChangeListeners.addAll(changeListenerSet); + public ConfigurationChangeCoordinator(Properties bootstrapProperties, RunMiNiFi runMiNiFi, + Set miNiFiConfigurationChangeListeners) { + this.bootstrapProperties = bootstrapProperties; + this.runMiNiFi = runMiNiFi; + this.configurationChangeListeners = Optional.ofNullable(miNiFiConfigurationChangeListeners).map(Collections::unmodifiableSet).orElse(Collections.emptySet()); } /** * Begins the associated notification service provided by the given implementation. In most implementations, no action will occur until this method is invoked. */ public void start() { + initialize(); changeIngestors.forEach(ChangeIngestor::start); } @@ -87,7 +72,7 @@ public class ConfigurationChangeCoordinator implements Closeable, ConfigurationC * Provide the mechanism by which listeners are notified */ public Collection notifyListeners(ByteBuffer newConfig) { - logger.info("Notifying Listeners of a change"); + LOGGER.info("Notifying Listeners of a change"); Collection listenerHandleResults = new ArrayList<>(configurationChangeListeners.size()); for (final ConfigurationChangeListener listener : getChangeListeners()) { @@ -99,16 +84,42 @@ public class ConfigurationChangeCoordinator implements Closeable, ConfigurationC result = new ListenerHandleResult(listener, ex); } listenerHandleResults.add(result); - logger.info("Listener notification result:" + result.toString()); + LOGGER.info("Listener notification result: {}", result); } return listenerHandleResults; } @Override - public void close() throws IOException { - for (ChangeIngestor changeIngestor : changeIngestors) { - changeIngestor.close(); + public void close() { + try { + for (ChangeIngestor changeIngestor : changeIngestors) { + changeIngestor.close(); + } + changeIngestors.clear(); + } catch (IOException e) { + LOGGER.warn("Could not successfully stop notifiers", e); + } + } + + private void initialize() { + close(); + // cleanup previously initialized ingestors + String ingestorsCsv = bootstrapProperties.getProperty(NOTIFIER_INGESTORS_KEY); + + if (ingestorsCsv != null && !ingestorsCsv.isEmpty()) { + for (String ingestorClassname : ingestorsCsv.split(",")) { + ingestorClassname = ingestorClassname.trim(); + try { + Class ingestorClass = Class.forName(ingestorClassname); + ChangeIngestor changeIngestor = (ChangeIngestor) ingestorClass.newInstance(); + changeIngestor.initialize(bootstrapProperties, runMiNiFi, this); + changeIngestors.add(changeIngestor); + LOGGER.info("Initialized ingestor: {}", ingestorClassname); + } catch (Exception e) { + LOGGER.error("Instantiating [{}] ingestor failed", ingestorClassname, e); + } + } } } } diff --git a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/differentiators/WholeConfigDifferentiator.java b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/differentiators/WholeConfigDifferentiator.java index 565a8f43d7..50c8c6b347 100644 --- a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/differentiators/WholeConfigDifferentiator.java +++ b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/differentiators/WholeConfigDifferentiator.java @@ -17,8 +17,8 @@ package org.apache.nifi.minifi.bootstrap.configuration.differentiators; -import org.apache.nifi.minifi.bootstrap.ConfigurationFileHolder; -import org.apache.nifi.minifi.bootstrap.configuration.differentiators.interfaces.Differentiator; +import org.apache.nifi.c2.client.api.ConfigurationFileHolder; +import org.apache.nifi.c2.client.api.Differentiator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/AbstractPullChangeIngestor.java b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/AbstractPullChangeIngestor.java index deebe90ce2..5071a68fc0 100644 --- a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/AbstractPullChangeIngestor.java +++ b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/AbstractPullChangeIngestor.java @@ -17,7 +17,7 @@ package org.apache.nifi.minifi.bootstrap.configuration.ingestors; -import org.apache.nifi.minifi.bootstrap.ConfigurationFileHolder; +import org.apache.nifi.c2.client.api.ConfigurationFileHolder; import org.apache.nifi.minifi.bootstrap.configuration.ConfigurationChangeNotifier; import org.apache.nifi.minifi.bootstrap.configuration.ingestors.interfaces.ChangeIngestor; import org.slf4j.Logger; diff --git a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/FileChangeIngestor.java b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/FileChangeIngestor.java index 39b272dbb6..b07166db4d 100644 --- a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/FileChangeIngestor.java +++ b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/FileChangeIngestor.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -16,15 +16,10 @@ */ package org.apache.nifi.minifi.bootstrap.configuration.ingestors; -import org.apache.commons.io.input.TeeInputStream; -import org.apache.commons.io.output.ByteArrayOutputStream; -import org.apache.nifi.minifi.bootstrap.ConfigurationFileHolder; -import org.apache.nifi.minifi.bootstrap.configuration.ConfigurationChangeNotifier; -import org.apache.nifi.minifi.bootstrap.configuration.differentiators.WholeConfigDifferentiator; -import org.apache.nifi.minifi.bootstrap.configuration.differentiators.interfaces.Differentiator; -import org.apache.nifi.minifi.bootstrap.configuration.ingestors.interfaces.ChangeIngestor; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import static java.nio.file.StandardWatchEventKinds.ENTRY_MODIFY; +import static java.util.Collections.emptyList; +import static org.apache.nifi.minifi.bootstrap.configuration.ConfigurationChangeCoordinator.NOTIFIER_INGESTORS_KEY; +import static org.apache.nifi.minifi.bootstrap.configuration.differentiators.WholeConfigDifferentiator.WHOLE_CONFIG_KEY; import java.io.FileInputStream; import java.io.IOException; @@ -39,16 +34,21 @@ import java.nio.file.WatchService; import java.util.Collections; import java.util.HashMap; import java.util.Map; +import java.util.Optional; import java.util.Properties; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; import java.util.function.Supplier; - -import static java.nio.file.StandardWatchEventKinds.ENTRY_MODIFY; -import static org.apache.nifi.minifi.bootstrap.configuration.ConfigurationChangeCoordinator.NOTIFIER_INGESTORS_KEY; -import static org.apache.nifi.minifi.bootstrap.configuration.differentiators.WholeConfigDifferentiator.WHOLE_CONFIG_KEY; +import org.apache.commons.io.input.TeeInputStream; +import org.apache.commons.io.output.ByteArrayOutputStream; +import org.apache.nifi.c2.client.api.ConfigurationFileHolder; +import org.apache.nifi.c2.client.api.Differentiator; +import org.apache.nifi.minifi.bootstrap.configuration.ConfigurationChangeNotifier; +import org.apache.nifi.minifi.bootstrap.configuration.differentiators.WholeConfigDifferentiator; +import org.apache.nifi.minifi.bootstrap.configuration.ingestors.interfaces.ChangeIngestor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * FileChangeIngestor provides a simple FileSystem monitor for detecting changes for a specified file as generated from its corresponding {@link Path}. Upon modifications to the associated file, @@ -98,29 +98,23 @@ public class FileChangeIngestor implements Runnable, ChangeIngestor { } protected boolean targetChanged() { - boolean targetChanged = false; + boolean targetChanged; - final WatchKey watchKey = this.watchService.poll(); + Optional watchKey = Optional.ofNullable(watchService.poll()); - if (watchKey == null) { - return targetChanged; - } - - for (WatchEvent watchEvt : watchKey.pollEvents()) { - final WatchEvent.Kind evtKind = watchEvt.kind(); - - final WatchEvent pathEvent = (WatchEvent) watchEvt; - final Path changedFile = pathEvent.context(); - - // determine target change by verifying if the changed file corresponds to the config file monitored for this path - targetChanged = (evtKind == ENTRY_MODIFY && changedFile.equals(configFilePath.getName(configFilePath.getNameCount() - 1))); - } + targetChanged = watchKey + .map(WatchKey::pollEvents) + .orElse(emptyList()) + .stream() + .anyMatch(watchEvent -> ENTRY_MODIFY == watchEvent.kind() + && ((WatchEvent) watchEvent).context().equals(configFilePath.getName(configFilePath.getNameCount() - 1))); // After completing inspection, reset for detection of subsequent change events - boolean valid = watchKey.reset(); - if (!valid) { - throw new IllegalStateException("Unable to reinitialize file system watcher."); - } + watchKey.map(WatchKey::reset) + .filter(valid -> !valid) + .ifPresent(valid -> { + throw new IllegalStateException("Unable to reinitialize file system watcher."); + }); return targetChanged; } @@ -212,14 +206,11 @@ public class FileChangeIngestor implements Runnable, ChangeIngestor { @Override public void start() { - executorService = Executors.newScheduledThreadPool(1, new ThreadFactory() { - @Override - public Thread newThread(final Runnable r) { - final Thread t = Executors.defaultThreadFactory().newThread(r); - t.setName("File Change Notifier Thread"); - t.setDaemon(true); - return t; - } + executorService = Executors.newScheduledThreadPool(1, r -> { + Thread t = Executors.defaultThreadFactory().newThread(r); + t.setName("File Change Notifier Thread"); + t.setDaemon(true); + return t; }); this.executorService.scheduleWithFixedDelay(this, 0, pollingSeconds, DEFAULT_POLLING_PERIOD_UNIT); } diff --git a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/PullHttpChangeIngestor.java b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/PullHttpChangeIngestor.java index f363dcd6e0..46056bc8c5 100644 --- a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/PullHttpChangeIngestor.java +++ b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/PullHttpChangeIngestor.java @@ -23,11 +23,11 @@ import okhttp3.OkHttpClient; import okhttp3.Request; import okhttp3.Response; import okhttp3.ResponseBody; -import org.apache.nifi.minifi.bootstrap.ConfigurationFileHolder; +import org.apache.nifi.c2.client.api.ConfigurationFileHolder; +import org.apache.nifi.c2.client.api.Differentiator; import org.apache.nifi.minifi.bootstrap.RunMiNiFi; import org.apache.nifi.minifi.bootstrap.configuration.ConfigurationChangeNotifier; import org.apache.nifi.minifi.bootstrap.configuration.differentiators.WholeConfigDifferentiator; -import org.apache.nifi.minifi.bootstrap.configuration.differentiators.interfaces.Differentiator; import org.apache.nifi.minifi.bootstrap.util.ByteBufferInputStream; import org.apache.nifi.minifi.commons.schema.ConfigSchema; import org.apache.nifi.minifi.commons.schema.SecurityPropertiesSchema; @@ -78,7 +78,7 @@ public class PullHttpChangeIngestor extends AbstractPullChangeIngestor { private static final String DEFAULT_CONNECT_TIMEOUT_MS = "5000"; private static final String DEFAULT_READ_TIMEOUT_MS = "15000"; - private static final String PULL_HTTP_BASE_KEY = NOTIFIER_INGESTORS_KEY + ".pull.http"; + public static final String PULL_HTTP_BASE_KEY = NOTIFIER_INGESTORS_KEY + ".pull.http"; public static final String PULL_HTTP_POLLING_PERIOD_KEY = PULL_HTTP_BASE_KEY + ".period.ms"; public static final String PORT_KEY = PULL_HTTP_BASE_KEY + ".port"; public static final String HOST_KEY = PULL_HTTP_BASE_KEY + ".hostname"; diff --git a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/RestChangeIngestor.java b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/RestChangeIngestor.java index d6b61d42d2..bf0f8f079a 100644 --- a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/RestChangeIngestor.java +++ b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/RestChangeIngestor.java @@ -19,11 +19,11 @@ package org.apache.nifi.minifi.bootstrap.configuration.ingestors; import org.apache.commons.io.input.TeeInputStream; import org.apache.commons.io.output.ByteArrayOutputStream; -import org.apache.nifi.minifi.bootstrap.ConfigurationFileHolder; +import org.apache.nifi.c2.client.api.ConfigurationFileHolder; +import org.apache.nifi.c2.client.api.Differentiator; import org.apache.nifi.minifi.bootstrap.configuration.ConfigurationChangeNotifier; import org.apache.nifi.minifi.bootstrap.configuration.ListenerHandleResult; import org.apache.nifi.minifi.bootstrap.configuration.differentiators.WholeConfigDifferentiator; -import org.apache.nifi.minifi.bootstrap.configuration.differentiators.interfaces.Differentiator; import org.apache.nifi.minifi.bootstrap.configuration.ingestors.interfaces.ChangeIngestor; import org.eclipse.jetty.server.Request; import org.eclipse.jetty.server.Server; diff --git a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/interfaces/ChangeIngestor.java b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/interfaces/ChangeIngestor.java index dcd39cd6d1..ca2d728bf6 100644 --- a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/interfaces/ChangeIngestor.java +++ b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/interfaces/ChangeIngestor.java @@ -17,7 +17,7 @@ package org.apache.nifi.minifi.bootstrap.configuration.ingestors.interfaces; -import org.apache.nifi.minifi.bootstrap.ConfigurationFileHolder; +import org.apache.nifi.c2.client.api.ConfigurationFileHolder; import org.apache.nifi.minifi.bootstrap.configuration.ConfigurationChangeNotifier; import java.io.IOException; diff --git a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/exception/StartupFailureException.java b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/exception/StartupFailureException.java new file mode 100644 index 0000000000..7c660ad88f --- /dev/null +++ b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/exception/StartupFailureException.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.minifi.bootstrap.exception; + +public class StartupFailureException extends RuntimeException { + + private static final long serialVersionUID = 1L; + + public StartupFailureException() { + } + + public StartupFailureException(String message) { + super(message); + } + + public StartupFailureException(String message, Throwable cause) { + super(message, cause); + } + + public StartupFailureException(Throwable cause) { + super(cause); + } + + public StartupFailureException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + } +} diff --git a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/BootstrapCodec.java b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/BootstrapCodec.java new file mode 100644 index 0000000000..124ff059f6 --- /dev/null +++ b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/BootstrapCodec.java @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.minifi.bootstrap.service; + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.util.Arrays; +import org.apache.nifi.minifi.bootstrap.RunMiNiFi; +import org.apache.nifi.minifi.bootstrap.exception.InvalidCommandException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class BootstrapCodec { + + private static final String TRUE = Boolean.TRUE.toString(); + private static final String FALSE = Boolean.FALSE.toString(); + + private final RunMiNiFi runner; + private final BufferedReader reader; + private final BufferedWriter writer; + private final Logger logger = LoggerFactory.getLogger(BootstrapCodec.class); + + public BootstrapCodec(RunMiNiFi runner, InputStream in, OutputStream out) { + this.runner = runner; + this.reader = new BufferedReader(new InputStreamReader(in)); + this.writer = new BufferedWriter(new OutputStreamWriter(out)); + } + + public void communicate() throws IOException { + String line = reader.readLine(); + String[] splits = line.split(" "); + if (splits.length == 0) { + throw new IOException("Received invalid command from MiNiFi: " + line); + } + + String cmd = splits[0]; + String[] args; + if (splits.length == 1) { + args = new String[0]; + } else { + args = Arrays.copyOfRange(splits, 1, splits.length); + } + + try { + processRequest(cmd, args); + } catch (InvalidCommandException exception) { + throw new IOException("Received invalid command from MiNiFi: " + line, exception); + } + } + + private void processRequest(String cmd, String[] args) throws InvalidCommandException, IOException { + switch (cmd) { + case "PORT": + handlePortCommand(args); + break; + case "STARTED": + handleStartedCommand(args); + break; + case "SHUTDOWN": + handleShutDownCommand(); + break; + case "RELOAD": + handleReloadCommand(); + break; + default: + throw new InvalidCommandException("Unknown command: " + cmd); + } + } + + private void handleReloadCommand() throws IOException { + logger.debug("Received 'RELOAD' command from MINIFI"); + writeOk(); + } + + private void handleShutDownCommand() throws IOException { + logger.debug("Received 'SHUTDOWN' command from MINIFI"); + runner.shutdownChangeNotifier(); + runner.getPeriodicStatusReporterManager().shutdownPeriodicStatusReporters(); + writeOk(); + } + + private void handleStartedCommand(String[] args) throws InvalidCommandException, IOException { + logger.debug("Received 'STARTED' command from MINIFI"); + if (args.length != 1) { + throw new InvalidCommandException("STARTED command must contain a status argument"); + } + + if (!TRUE.equalsIgnoreCase(args[0]) && !FALSE.equalsIgnoreCase(args[0])) { + throw new InvalidCommandException("Invalid status for STARTED command; should be true or false, but was '" + args[0] + "'"); + } + + runner.getPeriodicStatusReporterManager().shutdownPeriodicStatusReporters(); + runner.getPeriodicStatusReporterManager().startPeriodicNotifiers(); + runner.getConfigurationChangeCoordinator().start(); + + runner.setNiFiStarted(Boolean.parseBoolean(args[0])); + writeOk(); + } + + private void handlePortCommand(String[] args) throws InvalidCommandException, IOException { + logger.debug("Received 'PORT' command from MINIFI"); + if (args.length != 2) { + throw new InvalidCommandException("PORT command must contain the port and secretKey arguments"); + } + + int port; + try { + port = Integer.parseInt(args[0]); + } catch (NumberFormatException nfe) { + throw new InvalidCommandException("Invalid Port number; should be integer between 1 and 65535"); + } + + if (port < 1 || port > 65535) { + throw new InvalidCommandException("Invalid Port number; should be integer between 1 and 65535"); + } + + runner.setMiNiFiParameters(port, args[1]); + writeOk(); + } + + private void writeOk() throws IOException { + writer.write("OK"); + writer.newLine(); + writer.flush(); + } +} diff --git a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/BootstrapFileProvider.java b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/BootstrapFileProvider.java new file mode 100644 index 0000000000..67f57b4948 --- /dev/null +++ b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/BootstrapFileProvider.java @@ -0,0 +1,235 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.minifi.bootstrap.service; + +import static org.apache.nifi.minifi.bootstrap.RunMiNiFi.STATUS_FILE_PID_KEY; +import static org.apache.nifi.minifi.bootstrap.SensitiveProperty.SENSITIVE_PROPERTIES; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.attribute.PosixFilePermission; +import java.util.HashSet; +import java.util.Map; +import java.util.Optional; +import java.util.Properties; +import java.util.Set; +import java.util.stream.Collectors; +import org.apache.commons.lang3.StringUtils; +import org.apache.nifi.util.file.FileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class BootstrapFileProvider { + + private static final Logger LOGGER = LoggerFactory.getLogger(BootstrapFileProvider.class); + + private static final String MINIFI_PID_FILE_NAME = "minifi.pid"; + private static final String MINIFI_STATUS_FILE_NAME = "minifi.status"; + private static final String MINIFI_LOCK_FILE_NAME = "minifi.lock"; + private static final String DEFAULT_CONFIG_FILE = "./conf/bootstrap.conf"; + private static final String BOOTSTRAP_CONFIG_FILE_SYSTEM_PROPERTY_KEY = "org.apache.nifi.minifi.bootstrap.config.file"; + private static final String MINIFI_HOME_ENV_VARIABLE_KEY = "MINIFI_HOME"; + private static final String MINIFI_PID_DIR_PROP = "org.apache.nifi.minifi.bootstrap.config.pid.dir"; + private static final String DEFAULT_PID_DIR = "bin"; + + private final File bootstrapConfigFile; + + public BootstrapFileProvider(File bootstrapConfigFile) { + if (bootstrapConfigFile == null || !bootstrapConfigFile.exists()) { + throw new IllegalArgumentException("The specified bootstrap file doesn't exists: " + bootstrapConfigFile); + } + this.bootstrapConfigFile = bootstrapConfigFile; + } + + public static File getBootstrapConfFile() { + File bootstrapConfigFile = Optional.ofNullable(System.getProperty(BOOTSTRAP_CONFIG_FILE_SYSTEM_PROPERTY_KEY)) + .map(File::new) + .orElseGet(() -> Optional.ofNullable(System.getenv(MINIFI_HOME_ENV_VARIABLE_KEY)) + .map(File::new) + .map(nifiHomeFile -> new File(nifiHomeFile, DEFAULT_CONFIG_FILE)) + .orElseGet(() -> new File(DEFAULT_CONFIG_FILE))); + LOGGER.debug("Bootstrap config file: {}", bootstrapConfigFile); + return bootstrapConfigFile; + } + + public File getPidFile() throws IOException { + return getBootstrapFile(MINIFI_PID_FILE_NAME); + } + + public File getStatusFile() throws IOException { + return getBootstrapFile(MINIFI_STATUS_FILE_NAME); + } + + public File getLockFile() throws IOException { + return getBootstrapFile(MINIFI_LOCK_FILE_NAME); + } + + public File getReloadLockFile() { + File confDir = bootstrapConfigFile.getParentFile(); + File nifiHome = confDir.getParentFile(); + File bin = new File(nifiHome, "bin"); + File reloadFile = new File(bin, "minifi.reload.lock"); + + LOGGER.debug("Reload File: {}", reloadFile); + return reloadFile; + } + + public File getSwapFile() { + File confDir = bootstrapConfigFile.getParentFile(); + File swapFile = new File(confDir, "swap.yml"); + + LOGGER.debug("Swap File: {}", swapFile); + return swapFile; + } + + public Properties getBootstrapProperties() throws IOException { + if (!bootstrapConfigFile.exists()) { + throw new FileNotFoundException(bootstrapConfigFile.getAbsolutePath()); + } + + Properties bootstrapProperties = new Properties(); + try (FileInputStream fis = new FileInputStream(bootstrapConfigFile)) { + bootstrapProperties.load(fis); + } + + logProperties("Bootstrap", bootstrapProperties); + + return bootstrapProperties; + } + + public Properties getStatusProperties() { + Properties props = new Properties(); + + try { + File statusFile = getStatusFile(); + if (statusFile == null || !statusFile.exists()) { + LOGGER.debug("No status file to load properties from"); + return props; + } + + try (FileInputStream fis = new FileInputStream(statusFile)) { + props.load(fis); + } + } catch (IOException exception) { + LOGGER.error("Failed to load MiNiFi status properties"); + } + + logProperties("MiNiFi status", props); + + return props; + } + + public synchronized void saveStatusProperties(Properties minifiProps) throws IOException { + String pid = minifiProps.getProperty(STATUS_FILE_PID_KEY); + if (!StringUtils.isBlank(pid)) { + writePidFile(pid); + } + + File statusFile = getStatusFile(); + if (statusFile.exists() && !statusFile.delete()) { + LOGGER.warn("Failed to delete {}", statusFile); + } + + if (!statusFile.createNewFile()) { + throw new IOException("Failed to create file " + statusFile); + } + + try { + Set perms = new HashSet<>(); + perms.add(PosixFilePermission.OWNER_WRITE); + perms.add(PosixFilePermission.OWNER_READ); + perms.add(PosixFilePermission.GROUP_READ); + perms.add(PosixFilePermission.OTHERS_READ); + Files.setPosixFilePermissions(statusFile.toPath(), perms); + } catch (Exception e) { + LOGGER.warn("Failed to set permissions so that only the owner can read status file {}; " + + "this may allows others to have access to the key needed to communicate with MiNiFi. " + + "Permissions should be changed so that only the owner can read this file", statusFile); + } + + try (FileOutputStream fos = new FileOutputStream(statusFile)) { + minifiProps.store(fos, null); + fos.getFD().sync(); + } + + LOGGER.debug("Saving MiNiFi properties to {}", statusFile); + logProperties("Saved MiNiFi", minifiProps); + } + + private void writePidFile(String pid) throws IOException { + File pidFile = getPidFile(); + if (pidFile.exists() && !pidFile.delete()) { + LOGGER.warn("Failed to delete {}", pidFile); + } + + if (!pidFile.createNewFile()) { + throw new IOException("Failed to create file " + pidFile); + } + + try { + Set perms = new HashSet<>(); + perms.add(PosixFilePermission.OWNER_READ); + perms.add(PosixFilePermission.OWNER_WRITE); + Files.setPosixFilePermissions(pidFile.toPath(), perms); + } catch (Exception e) { + LOGGER.warn("Failed to set permissions so that only the owner can read pid file {}; " + + "this may allows others to have access to the key needed to communicate with MiNiFi. " + + "Permissions should be changed so that only the owner can read this file", pidFile); + } + + try (FileOutputStream fos = new FileOutputStream(pidFile)) { + fos.write(pid.getBytes(StandardCharsets.UTF_8)); + fos.getFD().sync(); + } + + LOGGER.debug("Saved Pid {} to {}", pid, pidFile); + } + + private File getBootstrapFile(String fileName) throws IOException { + File configFileDir = Optional.ofNullable(System.getProperty(MINIFI_PID_DIR_PROP)) + .map(String::trim) + .map(File::new) + .orElseGet(() -> { + File confDir = bootstrapConfigFile.getParentFile(); + File nifiHome = confDir.getParentFile(); + return new File(nifiHome, DEFAULT_PID_DIR); + }); + + FileUtils.ensureDirectoryExistAndCanAccess(configFileDir); + File statusFile = new File(configFileDir, fileName); + LOGGER.debug("Run File: {}", statusFile); + + return statusFile; + } + + private void logProperties(String type, Properties props) { + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("{} properties: {}", type, props.entrySet() + .stream() + .filter(e -> { + String key = ((String) e.getKey()).toLowerCase(); + return !SENSITIVE_PROPERTIES.contains(key); + }) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))); + } + } +} diff --git a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/CurrentPortProvider.java b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/CurrentPortProvider.java new file mode 100644 index 0000000000..e4d32dad53 --- /dev/null +++ b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/CurrentPortProvider.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.minifi.bootstrap.service; + +import static org.apache.nifi.minifi.bootstrap.RunMiNiFi.DEFAULT_LOGGER; +import static org.apache.nifi.minifi.bootstrap.RunMiNiFi.UNINITIALIZED; + +import org.apache.nifi.minifi.bootstrap.MiNiFiParameters; +import org.apache.nifi.minifi.bootstrap.util.UnixProcessUtils; + +public class CurrentPortProvider { + private final MiNiFiCommandSender miNiFiCommandSender; + private final MiNiFiParameters miNiFiParameters; + + public CurrentPortProvider(MiNiFiCommandSender miNiFiCommandSender, MiNiFiParameters miNiFiParameters) { + this.miNiFiCommandSender = miNiFiCommandSender; + this.miNiFiParameters = miNiFiParameters; + } + + public Integer getCurrentPort() { + int miNiFiPort = miNiFiParameters.getMiNiFiPort(); + if (miNiFiPort == UNINITIALIZED) { + DEFAULT_LOGGER.debug("Port is not defined"); + return null; + } + + DEFAULT_LOGGER.debug("Current port: {}", miNiFiPort); + + boolean success = miNiFiCommandSender.isPingSuccessful(miNiFiPort); + if (success) { + DEFAULT_LOGGER.debug("Successful PING on port {}", miNiFiPort); + return miNiFiPort; + } + + long minifiPid = miNiFiParameters.getMinifiPid(); + DEFAULT_LOGGER.debug("Current PID {}", minifiPid); + + boolean procRunning = UnixProcessUtils.isProcessRunning(minifiPid); + if (procRunning) { + return miNiFiPort; + } else { + return null; + } + } +} diff --git a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/GracefulShutdownParameterProvider.java b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/GracefulShutdownParameterProvider.java new file mode 100644 index 0000000000..4de7e673ec --- /dev/null +++ b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/GracefulShutdownParameterProvider.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.minifi.bootstrap.service; + +import java.io.IOException; +import java.util.Properties; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class GracefulShutdownParameterProvider { + private static final Logger LOGGER = LoggerFactory.getLogger(GracefulShutdownParameterProvider.class); + private static final String GRACEFUL_SHUTDOWN_PROP = "graceful.shutdown.seconds"; + private static final String DEFAULT_GRACEFUL_SHUTDOWN_VALUE = "20"; + private static final String INVALID_GRACEFUL_SHUTDOWN_SECONDS_MESSAGE = + "The {} property in Bootstrap Config File has an invalid value. Must be a non-negative integer, Falling back to the default {} value"; + + private final BootstrapFileProvider bootstrapFileProvider; + + public GracefulShutdownParameterProvider(BootstrapFileProvider bootstrapFileProvider) { + this.bootstrapFileProvider = bootstrapFileProvider; + } + + public int getGracefulShutdownSeconds() throws IOException { + Properties bootstrapProperties = bootstrapFileProvider.getBootstrapProperties(); + + String gracefulShutdown = bootstrapProperties.getProperty(GRACEFUL_SHUTDOWN_PROP, DEFAULT_GRACEFUL_SHUTDOWN_VALUE); + + int gracefulShutdownSeconds; + try { + gracefulShutdownSeconds = Integer.parseInt(gracefulShutdown); + } catch (NumberFormatException nfe) { + gracefulShutdownSeconds = Integer.parseInt(DEFAULT_GRACEFUL_SHUTDOWN_VALUE); + LOGGER.warn(INVALID_GRACEFUL_SHUTDOWN_SECONDS_MESSAGE, GRACEFUL_SHUTDOWN_PROP, gracefulShutdownSeconds); + } + + if (gracefulShutdownSeconds < 0) { + gracefulShutdownSeconds = Integer.parseInt(DEFAULT_GRACEFUL_SHUTDOWN_VALUE); + LOGGER.warn(INVALID_GRACEFUL_SHUTDOWN_SECONDS_MESSAGE, GRACEFUL_SHUTDOWN_PROP, gracefulShutdownSeconds); + } + return gracefulShutdownSeconds; + } +} diff --git a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/MiNiFiCommandSender.java b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/MiNiFiCommandSender.java new file mode 100644 index 0000000000..77595c046e --- /dev/null +++ b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/MiNiFiCommandSender.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.minifi.bootstrap.service; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.BufferedReader; +import java.io.EOFException; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.net.Socket; +import java.net.SocketTimeoutException; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Optional; +import java.util.stream.Collectors; +import org.apache.nifi.minifi.bootstrap.MiNiFiParameters; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class MiNiFiCommandSender { + + private static final Logger LOGGER = LoggerFactory.getLogger(MiNiFiCommandSender.class); + private static final String PING_CMD = "PING"; + private static final int SOCKET_TIMEOUT = 10000; + private static final int CONNECTION_TIMEOUT = 10000; + + private final MiNiFiParameters miNiFiParameters; + private final ObjectMapper objectMapper; + + public MiNiFiCommandSender(MiNiFiParameters miNiFiParameters, ObjectMapper objectMapper) { + this.miNiFiParameters = miNiFiParameters; + this.objectMapper = objectMapper; + } + + public Optional sendCommand(String cmd, Integer port, String... extraParams) throws IOException { + Optional response = Optional.empty(); + + if (port == null) { + LOGGER.info("Apache MiNiFi is not currently running"); + return response; + } + + try (Socket socket = new Socket()) { + LOGGER.debug("Connecting to MiNiFi instance"); + socket.setSoTimeout(SOCKET_TIMEOUT); + socket.connect(new InetSocketAddress("localhost", port), CONNECTION_TIMEOUT); + LOGGER.debug("Established connection to MiNiFi instance."); + + LOGGER.debug("Sending {} Command to port {}", cmd, port); + + String responseString; + try (OutputStream out = socket.getOutputStream()) { + out.write(getCommand(cmd, extraParams)); + out.flush(); + responseString = readResponse(socket); + } + + LOGGER.debug("Received response to {} command: {}", cmd, responseString); + response = Optional.of(responseString); + } catch (EOFException | SocketTimeoutException e) { + String message = "Failed to get response for " + cmd + " Potentially due to the process currently being down (restarting or otherwise)"; + throw new RuntimeException(message); + } + return response; + } + + T sendCommandForObject(String cmd, Integer port, Class clazz, String... extraParams) throws IOException { + return sendCommand(cmd, port, extraParams) + .map(response -> deserialize(cmd, response, clazz)) + .orElse(null); + } + + private String readResponse(Socket socket) throws IOException { + StringBuilder sb = new StringBuilder(); + int numLines = 0; + try (BufferedReader reader = new BufferedReader(new InputStreamReader(socket.getInputStream()))) { + String line; + while ((line = reader.readLine()) != null) { + if (numLines++ > 0) { + sb.append("\n"); + } + sb.append(line); + } + } + + return sb.toString().trim(); + } + + private byte[] getCommand(String cmd, String... args) { + String argsString = Arrays.stream(args).collect(Collectors.joining(" ")); + String commandWithArgs = cmd + " " + miNiFiParameters.getSecretKey() + (args.length > 0 ? " " : "") + argsString + "\n"; + return commandWithArgs.getBytes(StandardCharsets.UTF_8); + } + + private T deserialize(String cmd, String obj, Class clazz) { + T response; + try { + response = objectMapper.readValue(obj, clazz); + } catch (JsonProcessingException e) { + String message = "Failed to deserialize " + cmd + " response"; + LOGGER.error(message); + throw new RuntimeException(message, e); + } + return response; + } + + public boolean isPingSuccessful(int port) { + try { + return sendCommand(PING_CMD, port).filter(PING_CMD::equals).isPresent(); + } catch (IOException ioe) { + return false; + } + } +} diff --git a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/MiNiFiConfigurationChangeListener.java b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/MiNiFiConfigurationChangeListener.java new file mode 100644 index 0000000000..e1c30d835d --- /dev/null +++ b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/MiNiFiConfigurationChangeListener.java @@ -0,0 +1,206 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.minifi.bootstrap.service; + +import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; +import static org.apache.nifi.minifi.bootstrap.RunMiNiFi.CONF_DIR_KEY; +import static org.apache.nifi.minifi.bootstrap.RunMiNiFi.MINIFI_CONFIG_FILE_KEY; +import static org.apache.nifi.minifi.bootstrap.configuration.ingestors.PullHttpChangeIngestor.OVERRIDE_SECURITY; +import static org.apache.nifi.minifi.bootstrap.configuration.ingestors.PullHttpChangeIngestor.PULL_HTTP_BASE_KEY; +import static org.apache.nifi.minifi.bootstrap.util.ConfigTransformer.generateConfigFiles; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileWriter; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.nio.file.Files; +import java.util.Properties; +import java.util.concurrent.locks.ReentrantLock; +import org.apache.commons.io.IOUtils; +import org.apache.nifi.minifi.bootstrap.RunMiNiFi; +import org.apache.nifi.minifi.bootstrap.configuration.ConfigurationChangeException; +import org.apache.nifi.minifi.bootstrap.configuration.ConfigurationChangeListener; +import org.apache.nifi.minifi.bootstrap.util.ByteBufferInputStream; +import org.apache.nifi.minifi.bootstrap.util.ConfigTransformer; +import org.apache.nifi.minifi.commons.schema.ConfigSchema; +import org.apache.nifi.minifi.commons.schema.common.ConvertableSchema; +import org.apache.nifi.minifi.commons.schema.serialization.SchemaLoader; +import org.slf4j.Logger; + +public class MiNiFiConfigurationChangeListener implements ConfigurationChangeListener { + + private final RunMiNiFi runner; + private final Logger logger; + private final BootstrapFileProvider bootstrapFileProvider; + + private static final ReentrantLock handlingLock = new ReentrantLock(); + + public MiNiFiConfigurationChangeListener(RunMiNiFi runner, Logger logger, BootstrapFileProvider bootstrapFileProvider) { + this.runner = runner; + this.logger = logger; + this.bootstrapFileProvider = bootstrapFileProvider; + } + + @Override + public void handleChange(InputStream configInputStream) throws ConfigurationChangeException { + logger.info("Received notification of a change"); + + if (!handlingLock.tryLock()) { + throw new ConfigurationChangeException("Instance is already handling another change"); + } + // Store the incoming stream as a byte array to be shared among components that need it + try(ByteArrayOutputStream bufferedConfigOs = new ByteArrayOutputStream()) { + + Properties bootstrapProperties = bootstrapFileProvider.getBootstrapProperties(); + File configFile = new File(bootstrapProperties.getProperty(MINIFI_CONFIG_FILE_KEY)); + + IOUtils.copy(configInputStream, bufferedConfigOs); + + File swapConfigFile = bootstrapFileProvider.getSwapFile(); + logger.info("Persisting old configuration to {}", swapConfigFile.getAbsolutePath()); + + try (FileInputStream configFileInputStream = new FileInputStream(configFile)) { + Files.copy(configFileInputStream, swapConfigFile.toPath(), REPLACE_EXISTING); + } + + persistBackNonFlowSectionsFromOriginalSchema(bufferedConfigOs.toByteArray(), bootstrapProperties, configFile); + + // Create an input stream to feed to the config transformer + try (FileInputStream newConfigIs = new FileInputStream(configFile)) { + + try { + String confDir = bootstrapProperties.getProperty(CONF_DIR_KEY); + transformConfigurationFiles(confDir, newConfigIs, configFile, swapConfigFile); + } catch (Exception e) { + logger.debug("Transformation of new config file failed after swap file was created, deleting it."); + if (!swapConfigFile.delete()) { + logger.warn("The swap file failed to delete after a failed handling of a change. It should be cleaned up manually."); + } + throw e; + } + } + } catch (Exception e) { + throw new ConfigurationChangeException("Unable to perform reload of received configuration change", e); + } finally { + IOUtils.closeQuietly(configInputStream); + handlingLock.unlock(); + } + } + + @Override + public String getDescriptor() { + return "MiNiFiConfigurationChangeListener"; + } + + private void transformConfigurationFiles(String confDir, FileInputStream newConfigIs, File configFile, File swapConfigFile) throws Exception { + try { + logger.info("Performing transformation for input and saving outputs to {}", confDir); + ByteBuffer tempConfigFile = generateConfigFiles(newConfigIs, confDir, bootstrapFileProvider.getBootstrapProperties()); + runner.getConfigFileReference().set(tempConfigFile.asReadOnlyBuffer()); + reloadNewConfiguration(swapConfigFile, confDir); + } catch (Exception e) { + logger.debug("Transformation of new config file failed after replacing original with the swap file, reverting."); + try (FileInputStream swapConfigFileStream = new FileInputStream(swapConfigFile)) { + Files.copy(swapConfigFileStream, configFile.toPath(), REPLACE_EXISTING); + } + throw e; + } + } + + private void reloadNewConfiguration(File swapConfigFile, String confDir) throws Exception { + try { + logger.info("Reloading instance with new configuration"); + restartInstance(); + } catch (Exception e) { + logger.debug("Transformation of new config file failed after transformation into Flow.xml and nifi.properties, reverting."); + try (FileInputStream swapConfigFileStream = new FileInputStream(swapConfigFile)) { + ByteBuffer resetConfigFile = generateConfigFiles(swapConfigFileStream, confDir, bootstrapFileProvider.getBootstrapProperties()); + runner.getConfigFileReference().set(resetConfigFile.asReadOnlyBuffer()); + } + throw e; + } + } + + private void restartInstance() throws IOException { + try { + runner.reload(); + } catch (IOException e) { + throw new IOException("Unable to successfully restart MiNiFi instance after configuration change.", e); + } + } + + private void persistBackNonFlowSectionsFromOriginalSchema(byte[] newSchema, Properties bootstrapProperties, File configFile) { + try { + ConvertableSchema schemaNew = ConfigTransformer + .throwIfInvalid(SchemaLoader.loadConvertableSchemaFromYaml(new ByteArrayInputStream(newSchema))); + ConfigSchema configSchemaNew = ConfigTransformer.throwIfInvalid(schemaNew.convert()); + ConvertableSchema schemaOld = ConfigTransformer + .throwIfInvalid(SchemaLoader.loadConvertableSchemaFromYaml(new ByteBufferInputStream(runner.getConfigFileReference().get().duplicate()))); + ConfigSchema configSchemaOld = ConfigTransformer.throwIfInvalid(schemaOld.convert()); + + configSchemaNew.setNifiPropertiesOverrides(configSchemaOld.getNifiPropertiesOverrides()); + + if (!overrideCoreProperties(bootstrapProperties)) { + logger.debug("Preserving previous core properties..."); + configSchemaNew.setCoreProperties(configSchemaOld.getCoreProperties()); + } + + if (!overrideSecurityProperties(bootstrapProperties)) { + logger.debug("Preserving previous security properties..."); + configSchemaNew.setSecurityProperties(configSchemaOld.getSecurityProperties()); + } + + logger.debug("Persisting changes to {}", configFile.getAbsolutePath()); + SchemaLoader.toYaml(configSchemaNew, new FileWriter(configFile)); + } catch (Exception e) { + logger.error("Loading the old and the new schema for merging was not successful", e); + } + } + + private static boolean overrideSecurityProperties(Properties properties) { + String overrideSecurityProperties = (String) properties.getOrDefault(OVERRIDE_SECURITY, "false"); + boolean overrideSecurity; + if ("true".equalsIgnoreCase(overrideSecurityProperties) || "false".equalsIgnoreCase(overrideSecurityProperties)) { + overrideSecurity = Boolean.parseBoolean(overrideSecurityProperties); + } else { + throw new IllegalArgumentException( + "Property, " + OVERRIDE_SECURITY + ", to specify whether to override security properties must either be a value boolean value (\"true\" or \"false\")" + + " or left to the default value of \"false\". It is set to \"" + overrideSecurityProperties + "\"."); + } + + return overrideSecurity; + } + + private static boolean overrideCoreProperties(Properties properties) { + String overrideCorePropertiesKey = PULL_HTTP_BASE_KEY + ".override.core"; + String overrideCoreProps = (String) properties.getOrDefault(overrideCorePropertiesKey, "false"); + boolean overrideCoreProperties; + if ("true".equalsIgnoreCase(overrideCoreProps) || "false".equalsIgnoreCase(overrideCoreProps)) { + overrideCoreProperties = Boolean.parseBoolean(overrideCoreProps); + } else { + throw new IllegalArgumentException( + "Property, " + overrideCorePropertiesKey + ", to specify whether to override core properties must either be a value boolean value (\"true\" or \"false\")" + + " or left to the default value of \"false\". It is set to \"" + overrideCoreProps + "\"."); + } + + return overrideCoreProperties; + } +} diff --git a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/MiNiFiExecCommandProvider.java b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/MiNiFiExecCommandProvider.java new file mode 100644 index 0000000000..596dbbafbd --- /dev/null +++ b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/MiNiFiExecCommandProvider.java @@ -0,0 +1,158 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.minifi.bootstrap.service; + +import static org.apache.nifi.minifi.bootstrap.RunMiNiFi.CONF_DIR_KEY; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map.Entry; +import java.util.Optional; +import java.util.Properties; + +public class MiNiFiExecCommandProvider { + + private static final String DEFAULT_JAVA_CMD = "java"; + private static final String DEFAULT_LOG_DIR = "./logs"; + private static final String DEFAULT_LIB_DIR = "./lib"; + private static final String DEFAULT_CONF_DIR = "./conf"; + private static final String DEFAULT_CONFIG_FILE = DEFAULT_CONF_DIR + "/bootstrap.conf"; + private static final String WINDOWS_FILE_EXTENSION = ".exe"; + + private final BootstrapFileProvider bootstrapFileProvider; + + public MiNiFiExecCommandProvider(BootstrapFileProvider bootstrapFileProvider) { + this.bootstrapFileProvider = bootstrapFileProvider; + } + + /** + * Returns the process arguments required for the bootstrap to start the MiNiFi process. + * + * @param listenPort the port where the Bootstrap process is listening + * @param workingDir working dir of the MiNiFi + * @return the list of arguments to start the process + * @throws IOException throws IOException if any of the configuration file read fails + */ + public List getMiNiFiExecCommand(int listenPort, File workingDir) throws IOException { + Properties props = bootstrapFileProvider.getBootstrapProperties(); + File confDir = getFile(props.getProperty(CONF_DIR_KEY, DEFAULT_CONF_DIR).trim(), workingDir); + File libDir = getFile(props.getProperty("lib.dir", DEFAULT_LIB_DIR).trim(), workingDir); + String minifiLogDir = System.getProperty("org.apache.nifi.minifi.bootstrap.config.log.dir", DEFAULT_LOG_DIR).trim(); + + List cmd = new ArrayList<>(); + cmd.add(getJavaCommand(props)); + cmd.add("-classpath"); + cmd.add(buildClassPath(props, confDir, libDir)); + cmd.addAll(getJavaAdditionalArgs(props)); + cmd.add("-Dnifi.properties.file.path=" + getMiNiFiPropsFileName(props, confDir)); + cmd.add("-Dnifi.bootstrap.listen.port=" + listenPort); + cmd.add("-Dapp=MiNiFi"); + cmd.add("-Dorg.apache.nifi.minifi.bootstrap.config.log.dir=" + minifiLogDir); + cmd.add("org.apache.nifi.minifi.MiNiFi"); + + return cmd; + } + + private String getJavaCommand(Properties props) { + String javaCmd = props.getProperty("java"); + if (javaCmd == null) { + javaCmd = DEFAULT_JAVA_CMD; + } + if (javaCmd.equals(DEFAULT_JAVA_CMD)) { + Optional.ofNullable(System.getenv("JAVA_HOME")) + .map(javaHome -> getJavaCommandBasedOnExtension(javaHome, WINDOWS_FILE_EXTENSION) + .orElseGet(() -> getJavaCommandBasedOnExtension(javaHome, "").orElse(DEFAULT_JAVA_CMD))); + } + return javaCmd; + } + + private Optional getJavaCommandBasedOnExtension(String javaHome, String extension) { + String javaCmd = null; + File javaFile = new File(javaHome + File.separatorChar + "bin" + File.separatorChar + "java" + extension); + if (javaFile.exists() && javaFile.canExecute()) { + javaCmd = javaFile.getAbsolutePath(); + } + return Optional.ofNullable(javaCmd); + } + + private String buildClassPath(Properties props, File confDir, File libDir) { + + File[] libFiles = libDir.listFiles((dir, filename) -> filename.toLowerCase().endsWith(".jar")); + if (libFiles == null || libFiles.length == 0) { + throw new RuntimeException("Could not find lib directory at " + libDir.getAbsolutePath()); + } + + File[] confFiles = confDir.listFiles(); + if (confFiles == null || confFiles.length == 0) { + throw new RuntimeException("Could not find conf directory at " + confDir.getAbsolutePath()); + } + + List cpFiles = new ArrayList<>(confFiles.length + libFiles.length); + cpFiles.add(confDir.getAbsolutePath()); + for (File file : libFiles) { + cpFiles.add(file.getAbsolutePath()); + } + + StringBuilder classPathBuilder = new StringBuilder(); + for (int i = 0; i < cpFiles.size(); i++) { + String filename = cpFiles.get(i); + classPathBuilder.append(filename); + if (i < cpFiles.size() - 1) { + classPathBuilder.append(File.pathSeparatorChar); + } + } + + return classPathBuilder.toString(); + } + + private List getJavaAdditionalArgs(Properties props) { + List javaAdditionalArgs = new ArrayList<>(); + for (Entry entry : props.entrySet()) { + String key = (String) entry.getKey(); + String value = (String) entry.getValue(); + + if (key.startsWith("java.arg")) { + javaAdditionalArgs.add(value); + } + } + return javaAdditionalArgs; + } + + private String getMiNiFiPropsFileName(Properties props, File confDir) { + String minifiPropsFilename = props.getProperty("props.file"); + if (minifiPropsFilename == null) { + if (confDir.exists()) { + minifiPropsFilename = new File(confDir, "nifi.properties").getAbsolutePath(); + } else { + minifiPropsFilename = DEFAULT_CONFIG_FILE; + } + } + + return minifiPropsFilename.trim(); + } + + private File getFile(String filename, File workingDir) { + File file = new File(filename); + if (!file.isAbsolute()) { + file = new File(workingDir, filename); + } + return file; + } +} diff --git a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/MiNiFiListener.java b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/MiNiFiListener.java new file mode 100644 index 0000000000..e9a8eaff64 --- /dev/null +++ b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/MiNiFiListener.java @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.minifi.bootstrap.service; + +import java.io.IOException; +import java.io.InputStream; +import java.net.InetSocketAddress; +import java.net.ServerSocket; +import java.net.Socket; +import java.util.Optional; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import org.apache.nifi.minifi.bootstrap.RunMiNiFi; +import org.apache.nifi.minifi.bootstrap.util.LimitingInputStream; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class MiNiFiListener { + + private static final Logger LOGGER = LoggerFactory.getLogger(MiNiFiListener.class); + + private Listener listener; + private ServerSocket serverSocket; + + public int start(RunMiNiFi runner) throws IOException { + serverSocket = new ServerSocket(); + serverSocket.bind(new InetSocketAddress("localhost", 0)); + + listener = new Listener(serverSocket, runner); + Thread listenThread = new Thread(listener); + listenThread.setName("MiNiFi listener"); + listenThread.setDaemon(true); + listenThread.start(); + return serverSocket.getLocalPort(); + } + + public void stop() { + try { + if (serverSocket != null) { + serverSocket.close(); + } + } catch (IOException e) { + LOGGER.error("Failed to close socket"); + } + Optional.ofNullable(listener).ifPresent(Listener::stop); + } + + private static class Listener implements Runnable { + + private final ServerSocket serverSocket; + private final ExecutorService executor; + private final RunMiNiFi runner; + private volatile boolean stopped = false; + + public Listener(ServerSocket serverSocket, RunMiNiFi runner) { + this.serverSocket = serverSocket; + this.executor = Executors.newFixedThreadPool(2, runnable -> { + Thread t = Executors.defaultThreadFactory().newThread(runnable); + t.setDaemon(true); + t.setName("MiNiFi Bootstrap Command Listener"); + return t; + }); + + this.runner = runner; + } + + public void stop() { + stopped = true; + + try { + executor.shutdown(); + try { + if (!executor.awaitTermination(3, TimeUnit.SECONDS)) { + executor.shutdownNow(); + } + } catch (InterruptedException e) { + LOGGER.warn("Failed to stop the MiNiFi listener executor", e); + executor.shutdownNow(); + } + + serverSocket.close(); + } catch (IOException e) { + LOGGER.warn("Failed to close socket", e); + } catch (Exception e) { + LOGGER.warn("Failed to stop the MiNiFi listener executor", e); + } + } + + @Override + public void run() { + while (!serverSocket.isClosed()) { + try { + if (stopped) { + return; + } + + Socket socket; + try { + socket = serverSocket.accept(); + } catch (IOException ioe) { + if (stopped) { + return; + } + throw ioe; + } + + executor.submit(() -> { + // we want to ensure that we don't try to read data from an InputStream directly + // by a BufferedReader because any user on the system could open a socket and send + // a multi-gigabyte file without any new lines in order to crash the Bootstrap, + // which in turn may cause the Shutdown Hook to shutdown MiNiFi. + // So we will limit the amount of data to read to 4 KB + try (InputStream limitingIn = new LimitingInputStream(socket.getInputStream(), 4096)) { + BootstrapCodec codec = new BootstrapCodec(runner, limitingIn, socket.getOutputStream()); + codec.communicate(); + } catch (Exception t) { + LOGGER.error("Failed to communicate with MiNiFi due to exception: ", t); + } finally { + try { + socket.close(); + } catch (IOException ioe) { + LOGGER.warn("Failed to close the socket ", ioe); + } + } + }); + } catch (Exception t) { + LOGGER.error("Failed to receive information from MiNiFi due to exception: ", t); + } + } + } + } +} diff --git a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/MiNiFiStatusProvider.java b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/MiNiFiStatusProvider.java new file mode 100644 index 0000000000..acd6395ea8 --- /dev/null +++ b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/MiNiFiStatusProvider.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.minifi.bootstrap.service; + +import static org.apache.nifi.minifi.bootstrap.RunMiNiFi.UNINITIALIZED; +import static org.apache.nifi.minifi.bootstrap.util.UnixProcessUtils.isProcessRunning; + +import org.apache.nifi.minifi.bootstrap.MiNiFiStatus; + +public class MiNiFiStatusProvider { + + private final MiNiFiCommandSender miNiFiCommandSender; + + public MiNiFiStatusProvider(MiNiFiCommandSender miNiFiCommandSender) { + this.miNiFiCommandSender = miNiFiCommandSender; + } + + public MiNiFiStatus getStatus(int port, long pid) { + if (port == UNINITIALIZED && pid == UNINITIALIZED) { + return new MiNiFiStatus(); + } + + boolean pingSuccess = false; + if (port != UNINITIALIZED) { + pingSuccess = miNiFiCommandSender.isPingSuccessful(port); + } + + if (pingSuccess) { + return new MiNiFiStatus(port, pid, true, true); + } + + return new MiNiFiStatus(port, pid, false, isProcessRunning(pid)); + } +} diff --git a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/MiNiFiStdLogHandler.java b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/MiNiFiStdLogHandler.java new file mode 100644 index 0000000000..b134ae7dfe --- /dev/null +++ b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/MiNiFiStdLogHandler.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.minifi.bootstrap.service; + +import static org.apache.nifi.minifi.bootstrap.service.MiNiFiStdLogHandler.LoggerType.ERROR; +import static org.apache.nifi.minifi.bootstrap.service.MiNiFiStdLogHandler.LoggerType.STDOUT; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.HashSet; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.stream.Stream; +import org.jetbrains.annotations.NotNull; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class MiNiFiStdLogHandler { + + private static final Logger LOGGER = LoggerFactory.getLogger(MiNiFiStdLogHandler.class); + private static final String READ_FAILURE_MESSAGE = "Failed to read from MiNiFi's Standard {} stream"; + private static final String EXCEPTION_MESSAGE = "Exception: "; + + private final ExecutorService loggingExecutor; + private Set> loggingFutures; + + public MiNiFiStdLogHandler() { + loggingExecutor = Executors.newFixedThreadPool(2, runnable -> { + Thread t = Executors.defaultThreadFactory().newThread(runnable); + t.setDaemon(true); + t.setName("MiNiFi logging handler"); + return t; + }); + } + + public void initLogging(Process process) { + LOGGER.debug("Initializing MiNiFi's standard output/error loggers..."); + Optional.ofNullable(loggingFutures) + .map(Set::stream) + .orElse(Stream.empty()) + .forEach(future -> future.cancel(false)); + + Set> futures = new HashSet<>(); + futures.add(getFuture(process.getInputStream(), STDOUT)); + futures.add(getFuture(process.getErrorStream(), ERROR)); + loggingFutures = futures; + } + + @NotNull + private Future getFuture(InputStream in, LoggerType loggerType) { + return loggingExecutor.submit(() -> { + Logger logger = LoggerFactory.getLogger(loggerType.getLoggerName()); + try (BufferedReader reader = new BufferedReader(new InputStreamReader(in))) { + String line; + while ((line = reader.readLine()) != null) { + if (loggerType == ERROR) { + logger.error(line); + } else { + logger.info(line); + } + } + } catch (IOException e) { + LOGGER.warn(READ_FAILURE_MESSAGE, loggerType.getDisplayName()); + LOGGER.warn(EXCEPTION_MESSAGE, e); + } + }); + } + + public void shutdown() { + LOGGER.debug("Shutting down MiNiFi's standard output/error loggers..."); + loggingExecutor.shutdown(); + } + + enum LoggerType { + STDOUT("Output", "org.apache.nifi.minifi.StdOut"), + ERROR("Error", "org.apache.nifi.minifi.StdErr"); + + final String displayName; + final String loggerName; + + LoggerType(String displayName, String loggerName) { + this.displayName = displayName; + this.loggerName = loggerName; + } + + public String getDisplayName() { + return displayName; + } + + public String getLoggerName() { + return loggerName; + } + } +} diff --git a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/PeriodicStatusReporterManager.java b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/PeriodicStatusReporterManager.java new file mode 100644 index 0000000000..9086ca8045 --- /dev/null +++ b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/PeriodicStatusReporterManager.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.minifi.bootstrap.service; + +import static org.apache.nifi.minifi.commons.schema.common.BootstrapPropertyKeys.STATUS_REPORTER_COMPONENTS_KEY; + +import java.util.Collections; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Properties; +import java.util.Set; +import org.apache.nifi.minifi.bootstrap.MiNiFiParameters; +import org.apache.nifi.minifi.bootstrap.MiNiFiStatus; +import org.apache.nifi.minifi.bootstrap.QueryableStatusAggregator; +import org.apache.nifi.minifi.bootstrap.status.PeriodicStatusReporter; +import org.apache.nifi.minifi.commons.status.FlowStatusReport; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class PeriodicStatusReporterManager implements QueryableStatusAggregator { + private static final Logger LOGGER = LoggerFactory.getLogger(PeriodicStatusReporterManager.class); + private static final String FLOW_STATUS_REPORT_CMD = "FLOW_STATUS_REPORT"; + + private final Properties bootstrapProperties; + private final MiNiFiStatusProvider miNiFiStatusProvider; + private final MiNiFiCommandSender miNiFiCommandSender; + private final MiNiFiParameters miNiFiParameters; + + private Set periodicStatusReporters = Collections.emptySet(); + + public PeriodicStatusReporterManager(Properties bootstrapProperties, MiNiFiStatusProvider miNiFiStatusProvider, MiNiFiCommandSender miNiFiCommandSender, + MiNiFiParameters miNiFiParameters) { + this.bootstrapProperties = bootstrapProperties; + this.miNiFiStatusProvider = miNiFiStatusProvider; + this.miNiFiCommandSender = miNiFiCommandSender; + this.miNiFiParameters = miNiFiParameters; + } + + public void startPeriodicNotifiers() { + periodicStatusReporters = initializePeriodicNotifiers(); + + for (PeriodicStatusReporter periodicStatusReporter: periodicStatusReporters) { + periodicStatusReporter.start(); + LOGGER.debug("Started {} notifier", periodicStatusReporter.getClass().getCanonicalName()); + } + } + + public void shutdownPeriodicStatusReporters() { + LOGGER.debug("Initiating shutdown of bootstrap periodic status reporters..."); + for (PeriodicStatusReporter periodicStatusReporter : periodicStatusReporters) { + try { + periodicStatusReporter.stop(); + } catch (Exception exception) { + LOGGER.error("Could not successfully stop periodic status reporter " + periodicStatusReporter.getClass() + " due to ", exception); + } + } + } + + public FlowStatusReport statusReport(String statusRequest) { + MiNiFiStatus status = miNiFiStatusProvider.getStatus(miNiFiParameters.getMiNiFiPort(), miNiFiParameters.getMinifiPid()); + + List problemsGeneratingReport = new LinkedList<>(); + if (!status.isProcessRunning()) { + problemsGeneratingReport.add("MiNiFi process is not running"); + } + + if (!status.isRespondingToPing()) { + problemsGeneratingReport.add("MiNiFi process is not responding to pings"); + } + + if (!problemsGeneratingReport.isEmpty()) { + FlowStatusReport flowStatusReport = new FlowStatusReport(); + flowStatusReport.setErrorsGeneratingReport(problemsGeneratingReport); + return flowStatusReport; + } + + return getFlowStatusReport(statusRequest, status.getPort()); + } + + private Set initializePeriodicNotifiers() { + LOGGER.debug("Initiating bootstrap periodic status reporters..."); + Set statusReporters = new HashSet<>(); + + String reportersCsv = bootstrapProperties.getProperty(STATUS_REPORTER_COMPONENTS_KEY); + + if (reportersCsv != null && !reportersCsv.isEmpty()) { + for (String reporterClassname : reportersCsv.split(",")) { + try { + Class reporterClass = Class.forName(reporterClassname); + PeriodicStatusReporter reporter = (PeriodicStatusReporter) reporterClass.newInstance(); + reporter.initialize(bootstrapProperties, this); + statusReporters.add(reporter); + LOGGER.debug("Initialized {} notifier", reporterClass.getCanonicalName()); + } catch (InstantiationException | IllegalAccessException | ClassNotFoundException e) { + throw new RuntimeException("Issue instantiating notifier " + reporterClassname, e); + } + } + } + return statusReporters; + } + + private FlowStatusReport getFlowStatusReport(String statusRequest, int port) { + FlowStatusReport flowStatusReport; + try { + flowStatusReport = miNiFiCommandSender.sendCommandForObject(FLOW_STATUS_REPORT_CMD, port, FlowStatusReport.class, statusRequest); + } catch (Exception e) { + flowStatusReport = new FlowStatusReport(); + String message = "Failed to get status report from MiNiFi due to:" + e.getMessage(); + flowStatusReport.setErrorsGeneratingReport(Collections.singletonList(message)); + LOGGER.error(message, e); + } + return flowStatusReport; + } + +} diff --git a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/ReloadService.java b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/ReloadService.java new file mode 100644 index 0000000000..dc87028037 --- /dev/null +++ b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/ReloadService.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.minifi.bootstrap.service; + +import static org.apache.nifi.minifi.bootstrap.RunMiNiFi.DEFAULT_LOGGER; +import static org.apache.nifi.minifi.bootstrap.RunMiNiFi.UNINITIALIZED; + +import java.io.File; +import java.io.IOException; +import java.util.Optional; +import org.apache.nifi.minifi.bootstrap.MiNiFiParameters; +import org.apache.nifi.minifi.bootstrap.RunMiNiFi; +import org.apache.nifi.minifi.bootstrap.util.UnixProcessUtils; + +public class ReloadService { + private final BootstrapFileProvider bootstrapFileProvider; + private final MiNiFiParameters miNiFiParameters; + private final MiNiFiCommandSender miNiFiCommandSender; + private static final String RELOAD_CMD = "RELOAD"; + private final CurrentPortProvider currentPortProvider; + private final GracefulShutdownParameterProvider gracefulShutdownParameterProvider; + private final RunMiNiFi runMiNiFi; + + public ReloadService(BootstrapFileProvider bootstrapFileProvider, MiNiFiParameters miNiFiParameters, + MiNiFiCommandSender miNiFiCommandSender, CurrentPortProvider currentPortProvider, + GracefulShutdownParameterProvider gracefulShutdownParameterProvider, RunMiNiFi runMiNiFi) { + this.bootstrapFileProvider = bootstrapFileProvider; + this.miNiFiParameters = miNiFiParameters; + this.miNiFiCommandSender = miNiFiCommandSender; + this.currentPortProvider = currentPortProvider; + this.gracefulShutdownParameterProvider = gracefulShutdownParameterProvider; + this.runMiNiFi = runMiNiFi; + } + + public void reload() throws IOException { + // indicate that a reload command is in progress + File reloadLockFile = bootstrapFileProvider.getReloadLockFile(); + if (!reloadLockFile.exists()) { + reloadLockFile.createNewFile(); + } + + long minifiPid = miNiFiParameters.getMinifiPid(); + try { + Optional commandResponse = miNiFiCommandSender.sendCommand(RELOAD_CMD, currentPortProvider.getCurrentPort()); + if (commandResponse.filter(RELOAD_CMD::equals).isPresent()) { + DEFAULT_LOGGER.info("Apache MiNiFi has accepted the Reload Command and is reloading"); + if (minifiPid != UNINITIALIZED) { + UnixProcessUtils.gracefulShutDownMiNiFiProcess(minifiPid, "MiNiFi has not finished shutting down after {} seconds as part of configuration reload. Killing process.", + gracefulShutdownParameterProvider.getGracefulShutdownSeconds()); + runMiNiFi.setReloading(true); + DEFAULT_LOGGER.info("MiNiFi has finished shutting down and will be reloaded."); + } + } else { + DEFAULT_LOGGER.error("When sending RELOAD command to MiNiFi, got unexpected response {}.", commandResponse.orElse(null)); + } + } catch (IOException e) { + if (minifiPid == UNINITIALIZED) { + DEFAULT_LOGGER.error("No PID found for the MiNiFi process, so unable to kill process; The process should be killed manually."); + } else { + DEFAULT_LOGGER.error("Will kill the MiNiFi Process with PID {}", minifiPid); + UnixProcessUtils.killProcessTree(minifiPid); + } + } finally { + if (reloadLockFile.exists() && !reloadLockFile.delete()) { + DEFAULT_LOGGER.error("Failed to delete reload lock file {}; this file should be cleaned up manually", reloadLockFile); + } + } + } +} diff --git a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformer.java b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformer.java index 04f903e192..ca8d6a54f3 100644 --- a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformer.java +++ b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformer.java @@ -17,6 +17,29 @@ package org.apache.nifi.minifi.bootstrap.util; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.security.SecureRandom; +import java.util.Arrays; +import java.util.Base64; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Properties; +import java.util.stream.Collectors; +import java.util.zip.GZIPOutputStream; +import javax.xml.transform.dom.DOMSource; +import javax.xml.transform.stream.StreamResult; +import org.apache.commons.io.input.TeeInputStream; import org.apache.commons.io.output.ByteArrayOutputStream; import org.apache.commons.lang3.StringUtils; import org.apache.nifi.minifi.bootstrap.configuration.ConfigurationChangeException; @@ -55,26 +78,6 @@ import org.w3c.dom.DOMException; import org.w3c.dom.Document; import org.w3c.dom.Element; -import javax.xml.transform.dom.DOMSource; -import javax.xml.transform.stream.StreamResult; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.security.SecureRandom; -import java.util.Arrays; -import java.util.Base64; -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import java.util.stream.Collectors; -import java.util.zip.GZIPOutputStream; - public final class ConfigTransformer { // Underlying version of NIFI will be using public static final String ROOT_GROUP = "Root-Group"; @@ -88,42 +91,68 @@ public final class ConfigTransformer { private ConfigTransformer() { } + public static ByteBuffer generateConfigFiles(InputStream configIs, String configDestinationPath, Properties bootstrapProperties) throws ConfigurationChangeException, IOException { + try (java.io.ByteArrayOutputStream byteArrayOutputStream = new java.io.ByteArrayOutputStream(); + TeeInputStream teeInputStream = new TeeInputStream(configIs, byteArrayOutputStream)) { + + ConfigTransformer.transformConfigFile( + teeInputStream, + configDestinationPath, + bootstrapProperties + ); + + return ByteBuffer.wrap(byteArrayOutputStream.toByteArray()); + } catch (ConfigurationChangeException e){ + throw e; + } catch (Exception e) { + throw new IOException("Unable to successfully transform the provided configuration", e); + } + } + public static void transformConfigFile(InputStream sourceStream, String destPath, Properties bootstrapProperties) throws Exception { - ConvertableSchema convertableSchema = throwIfInvalid(SchemaLoader.loadConvertableSchemaFromYaml(sourceStream)); - ConfigSchema configSchema = throwIfInvalid(convertableSchema.convert()); + ConvertableSchema convertableSchemaNew = throwIfInvalid(SchemaLoader.loadConvertableSchemaFromYaml(sourceStream)); + ConfigSchema configSchemaNew = throwIfInvalid(convertableSchemaNew.convert()); SecurityPropertiesSchema securityProperties = BootstrapTransformer.buildSecurityPropertiesFromBootstrap(bootstrapProperties).orElse(null); ProvenanceReportingSchema provenanceReportingProperties = BootstrapTransformer.buildProvenanceReportingPropertiesFromBootstrap(bootstrapProperties).orElse(null); // See if we are providing defined properties from the filesystem configurations and use those as the definitive values if (securityProperties != null) { - configSchema.setSecurityProperties(securityProperties); + configSchemaNew.setSecurityProperties(securityProperties); logger.info("Bootstrap flow override: Replaced security properties"); } + if (provenanceReportingProperties != null) { - configSchema.setProvenanceReportingProperties(provenanceReportingProperties); + configSchemaNew.setProvenanceReportingProperties(provenanceReportingProperties); logger.info("Bootstrap flow override: Replaced provenance reporting properties"); } // Replace all processor SSL controller services with MiNiFi parent, if bootstrap boolean is set to true if (BootstrapTransformer.processorSSLOverride(bootstrapProperties)) { - for (ProcessorSchema processorConfig : configSchema.getProcessGroupSchema().getProcessors()) { + for (ProcessorSchema processorConfig : configSchemaNew.getProcessGroupSchema().getProcessors()) { processorConfig.getProperties().replace("SSL Context Service", processorConfig.getProperties().get("SSL Context Service"), "SSL-Context-Service"); logger.info("Bootstrap flow override: Replaced {} SSL Context Service with parent MiNiFi SSL", processorConfig.getName()); } } + Optional.ofNullable(bootstrapProperties) + .map(Properties::entrySet) + .orElse(Collections.emptySet()) + .stream() + .filter(entry -> ((String) entry.getKey()).startsWith("c2")) + .forEach(entry -> configSchemaNew.getNifiPropertiesOverrides().putIfAbsent((String) entry.getKey(), (String) entry.getValue())); + // Create nifi.properties and flow.xml.gz in memory ByteArrayOutputStream nifiPropertiesOutputStream = new ByteArrayOutputStream(); - writeNiFiProperties(configSchema, nifiPropertiesOutputStream); + writeNiFiProperties(configSchemaNew, nifiPropertiesOutputStream); - writeFlowXmlFile(configSchema, destPath); + writeFlowXmlFile(configSchemaNew, destPath); // Write nifi.properties and flow.xml.gz writeNiFiPropertiesFile(nifiPropertiesOutputStream, destPath); } - private static T throwIfInvalid(T schema) throws InvalidConfigurationException { + public static T throwIfInvalid(T schema) throws InvalidConfigurationException { if (!schema.isValid()) { throw new InvalidConfigurationException("Failed to transform config file due to:[" + schema.getValidationIssues().stream().sorted().collect(Collectors.joining("], [")) + "]"); @@ -705,8 +734,8 @@ public final class ConfigTransformer { protected static void addPosition(final Element parentElement) { final Element element = parentElement.getOwnerDocument().createElement("position"); - element.setAttribute("x", String.valueOf("0")); - element.setAttribute("y", String.valueOf("0")); + element.setAttribute("x", "0"); + element.setAttribute("y", "0"); parentElement.appendChild(element); } diff --git a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/UnixProcessUtils.java b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/UnixProcessUtils.java new file mode 100644 index 0000000000..a6d13ca416 --- /dev/null +++ b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/UnixProcessUtils.java @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.minifi.bootstrap.util; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.Reader; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/* + * Utility class for providing information about the running MiNiFi process. + * The methods which are using the PID are working only on unix systems, and should be used only as a fallback in case the PING command fails. + * */ +public class UnixProcessUtils { + private static final Logger LOGGER = LoggerFactory.getLogger(UnixProcessUtils.class); + + public static boolean isProcessRunning(Long pid) { + if (pid == null) { + LOGGER.error("Unable to get process status due to missing process id"); + return false; + } + try { + // We use the "ps" command to check if the process is still running. + ProcessBuilder builder = new ProcessBuilder(); + String pidString = String.valueOf(pid); + + builder.command("ps", "-p", pidString); + Process proc = builder.start(); + + // Look for the pid in the output of the 'ps' command. + boolean running = false; + String line; + try (InputStream in = proc.getInputStream(); + Reader streamReader = new InputStreamReader(in); + BufferedReader reader = new BufferedReader(streamReader)) { + + while ((line = reader.readLine()) != null) { + if (line.trim().startsWith(pidString)) { + running = true; + } + } + } + + // If output of the ps command had our PID, the process is running. + LOGGER.debug("Process with PID {} is {}running", pid, running ? "" : "not "); + + return running; + } catch (IOException ioe) { + LOGGER.error("Failed to determine if Process {} is running; assuming that it is not", pid); + return false; + } + } + + public static void gracefulShutDownMiNiFiProcess(Long pid, String s, int gracefulShutdownSeconds) { + long startWait = System.nanoTime(); + while (UnixProcessUtils.isProcessRunning(pid)) { + LOGGER.info("Waiting for Apache MiNiFi to finish shutting down..."); + long waitNanos = System.nanoTime() - startWait; + long waitSeconds = TimeUnit.NANOSECONDS.toSeconds(waitNanos); + if (waitSeconds >= gracefulShutdownSeconds || gracefulShutdownSeconds == 0) { + if (UnixProcessUtils.isProcessRunning(pid)) { + LOGGER.warn(s, gracefulShutdownSeconds); + try { + UnixProcessUtils.killProcessTree(pid); + } catch (IOException ioe) { + LOGGER.error("Failed to kill Process with PID {}", pid); + } + } + break; + } else { + try { + Thread.sleep(2000L); + } catch (InterruptedException ie) { + } + } + } + } + + public static void killProcessTree(Long pid) throws IOException { + LOGGER.debug("Killing Process Tree for PID {}", pid); + + List children = getChildProcesses(pid); + LOGGER.debug("Children of PID {}: {}", pid, children); + + for (Long childPid : children) { + killProcessTree(childPid); + } + + Runtime.getRuntime().exec(new String[]{"kill", "-9", String.valueOf(pid)}); + } + + /** + * Checks the status of the given process. + * + * @param process the process object what we want to check + * @return true if the process is Alive + */ + public static boolean isAlive(Process process) { + try { + process.exitValue(); + return false; + } catch (IllegalStateException | IllegalThreadStateException itse) { + return true; + } + } + + private static List getChildProcesses(Long ppid) throws IOException { + Process proc = Runtime.getRuntime().exec(new String[]{"ps", "-o", "pid", "--no-headers", "--ppid", String.valueOf(ppid)}); + List childPids = new ArrayList<>(); + try (InputStream in = proc.getInputStream(); + BufferedReader reader = new BufferedReader(new InputStreamReader(in))) { + + String line; + while ((line = reader.readLine()) != null) { + try { + Long childPid = Long.valueOf(line.trim()); + childPids.add(childPid); + } catch (NumberFormatException e) { + LOGGER.trace("Failed to parse PID", e); + } + } + } + + return childPids; + } +} diff --git a/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/RunMiNiFiTest.java b/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/RunMiNiFiTest.java index 5acc99063c..82fcfae8ba 100644 --- a/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/RunMiNiFiTest.java +++ b/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/RunMiNiFiTest.java @@ -36,7 +36,6 @@ public class RunMiNiFiTest { @Test public void buildSecurityPropertiesNotDefined() throws Exception { - final RunMiNiFi testMiNiFi = new RunMiNiFi(null); final Properties bootstrapProperties = getTestBootstrapProperties("bootstrap-ssl-ctx/bootstrap.conf.default"); final Optional securityPropsOptional = BootstrapTransformer.buildSecurityPropertiesFromBootstrap(bootstrapProperties); assertFalse(securityPropsOptional.isPresent()); @@ -44,7 +43,6 @@ public class RunMiNiFiTest { @Test public void buildSecurityPropertiesDefined() throws Exception { - final RunMiNiFi testMiNiFi = new RunMiNiFi(null); final Properties bootstrapProperties = getTestBootstrapProperties("bootstrap-ssl-ctx/bootstrap.conf.configured"); final Optional securityPropsOptional = BootstrapTransformer.buildSecurityPropertiesFromBootstrap(bootstrapProperties); assertTrue(securityPropsOptional.isPresent()); @@ -73,7 +71,6 @@ public class RunMiNiFiTest { @Test public void buildSecurityPropertiesDefinedButInvalid() throws Exception { - final RunMiNiFi testMiNiFi = new RunMiNiFi(null); final Properties bootstrapProperties = getTestBootstrapProperties("bootstrap-ssl-ctx/bootstrap.conf.configured.invalid"); final Optional securityPropsOptional = BootstrapTransformer.buildSecurityPropertiesFromBootstrap(bootstrapProperties); assertTrue(securityPropsOptional.isPresent()); @@ -99,7 +96,6 @@ public class RunMiNiFiTest { @Test public void buildProvenanceReportingNotDefined() throws Exception { - final RunMiNiFi testMiNiFi = new RunMiNiFi(null); final Properties bootstrapProperties = getTestBootstrapProperties("bootstrap-provenance-reporting/bootstrap.conf.default"); final Optional provenanceReportingPropsOptional = BootstrapTransformer.buildProvenanceReportingPropertiesFromBootstrap(bootstrapProperties); assertFalse(provenanceReportingPropsOptional.isPresent()); @@ -107,7 +103,6 @@ public class RunMiNiFiTest { @Test public void buildProvenanceReportingDefined() throws Exception { - final RunMiNiFi testMiNiFi = new RunMiNiFi(null); final Properties bootstrapProperties = getTestBootstrapProperties("bootstrap-provenance-reporting/bootstrap.conf.configured"); final Optional provenanceReportingPropsOptional = BootstrapTransformer.buildProvenanceReportingPropertiesFromBootstrap(bootstrapProperties); assertTrue(provenanceReportingPropsOptional.isPresent()); diff --git a/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/configuration/ConfigurationChangeCoordinatorTest.java b/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/configuration/ConfigurationChangeCoordinatorTest.java deleted file mode 100644 index 75142bf75d..0000000000 --- a/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/configuration/ConfigurationChangeCoordinatorTest.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.nifi.minifi.bootstrap.configuration; - -import org.apache.nifi.minifi.bootstrap.ConfigurationFileHolder; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.mockito.Mockito; - -import java.io.InputStream; -import java.nio.ByteBuffer; -import java.util.Arrays; -import java.util.Collections; -import java.util.Properties; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.Mockito.verify; - -public class ConfigurationChangeCoordinatorTest { - - private ConfigurationChangeCoordinator coordinatorSpy; - private final Properties properties = new Properties(); - - @BeforeEach - public void setUp() { - coordinatorSpy = Mockito.spy(new ConfigurationChangeCoordinator()); - } - - @AfterEach - public void tearDown() throws Exception { - coordinatorSpy.close(); - } - - @Test - public void testInit() { - properties.put("nifi.minifi.notifier.ingestors", "org.apache.nifi.minifi.bootstrap.configuration.ingestors.RestChangeIngestor"); - final ConfigurationChangeListener testListener = Mockito.mock(ConfigurationChangeListener.class); - coordinatorSpy.initialize(properties, Mockito.mock(ConfigurationFileHolder.class), Collections.singleton(testListener)); - } - - @Test - public void testNotifyListeners() throws Exception { - final ConfigurationChangeListener testListener = Mockito.mock(ConfigurationChangeListener.class); - coordinatorSpy.initialize(properties, Mockito.mock(ConfigurationFileHolder.class), Collections.singleton(testListener)); - - assertEquals(coordinatorSpy.getChangeListeners().size(), 1, "Did not receive the correct number of registered listeners"); - - coordinatorSpy.notifyListeners(ByteBuffer.allocate(1)); - - verify(testListener, Mockito.atMost(1)).handleChange(Mockito.any(InputStream.class)); - } - - @Test - public void testRegisterListener() { - final ConfigurationChangeListener firstListener = Mockito.mock(ConfigurationChangeListener.class); - coordinatorSpy.initialize(properties, Mockito.mock(ConfigurationFileHolder.class), Collections.singleton(firstListener)); - - assertEquals(coordinatorSpy.getChangeListeners().size(), 1, "Did not receive the correct number of registered listeners"); - - coordinatorSpy.initialize(properties, Mockito.mock(ConfigurationFileHolder.class), Arrays.asList(firstListener, firstListener)); - assertEquals(coordinatorSpy.getChangeListeners().size(), 1, "Did not receive the correct number of registered listeners"); - - final ConfigurationChangeListener secondListener = Mockito.mock(ConfigurationChangeListener.class); - coordinatorSpy.initialize(properties, Mockito.mock(ConfigurationFileHolder.class), Arrays.asList(firstListener, secondListener)); - assertEquals(coordinatorSpy.getChangeListeners().size(), 2, "Did not receive the correct number of registered listeners"); - - } -} diff --git a/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/configuration/differentiators/WholeConfigDifferentiatorTest.java b/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/configuration/differentiators/WholeConfigDifferentiatorTest.java index a77d6c4b75..cd782d8515 100644 --- a/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/configuration/differentiators/WholeConfigDifferentiatorTest.java +++ b/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/configuration/differentiators/WholeConfigDifferentiatorTest.java @@ -17,13 +17,9 @@ package org.apache.nifi.minifi.bootstrap.configuration.differentiators; -import okhttp3.Request; -import org.apache.commons.io.FileUtils; -import org.apache.nifi.minifi.bootstrap.ConfigurationFileHolder; -import org.apache.nifi.minifi.bootstrap.configuration.differentiators.interfaces.Differentiator; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; -import org.mockito.Mockito; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.when; import java.io.FileInputStream; import java.io.IOException; @@ -33,10 +29,13 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.Properties; import java.util.concurrent.atomic.AtomicReference; - -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.Mockito.when; +import okhttp3.Request; +import org.apache.commons.io.FileUtils; +import org.apache.nifi.c2.client.api.ConfigurationFileHolder; +import org.apache.nifi.c2.client.api.Differentiator; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; public class WholeConfigDifferentiatorTest { diff --git a/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/FileChangeIngestorTest.java b/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/FileChangeIngestorTest.java index c0bc3e8456..38b1100369 100644 --- a/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/FileChangeIngestorTest.java +++ b/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/FileChangeIngestorTest.java @@ -28,13 +28,11 @@ import java.nio.file.Paths; import java.nio.file.WatchEvent; import java.nio.file.WatchKey; import java.nio.file.WatchService; -import java.util.Iterator; -import java.util.List; +import java.util.Collections; import java.util.Properties; - -import org.apache.nifi.minifi.bootstrap.ConfigurationFileHolder; +import org.apache.nifi.c2.client.api.ConfigurationFileHolder; +import org.apache.nifi.c2.client.api.Differentiator; import org.apache.nifi.minifi.bootstrap.configuration.ConfigurationChangeNotifier; -import org.apache.nifi.minifi.bootstrap.configuration.differentiators.interfaces.Differentiator; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -140,21 +138,14 @@ public class FileChangeIngestorTest { /* Helper methods to establish mock environment */ private WatchKey createMockWatchKeyForPath(String configFilePath) { - final WatchKey mockWatchKey = Mockito.mock(WatchKey.class); - final List> mockWatchEvents = (List>) Mockito.mock(List.class); - when(mockWatchKey.pollEvents()).thenReturn(mockWatchEvents); - when(mockWatchKey.reset()).thenReturn(true); - - final Iterator mockIterator = Mockito.mock(Iterator.class); - when(mockWatchEvents.iterator()).thenReturn(mockIterator); - - final WatchEvent mockWatchEvent = Mockito.mock(WatchEvent.class); - when(mockIterator.hasNext()).thenReturn(true, false); - when(mockIterator.next()).thenReturn(mockWatchEvent); + WatchKey mockWatchKey = Mockito.mock(WatchKey.class); + WatchEvent mockWatchEvent = Mockito.mock(WatchEvent.class); // In this case, we receive a trigger event for the directory monitored, and it was the file monitored when(mockWatchEvent.context()).thenReturn(Paths.get(configFilePath)); when(mockWatchEvent.kind()).thenReturn(ENTRY_MODIFY); + when(mockWatchKey.pollEvents()).thenReturn(Collections.singletonList(mockWatchEvent)); + when(mockWatchKey.reset()).thenReturn(true); return mockWatchKey; } diff --git a/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/PullHttpChangeIngestorSSLTest.java b/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/PullHttpChangeIngestorSSLTest.java index 1a4df6ff5c..bc546fe552 100644 --- a/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/PullHttpChangeIngestorSSLTest.java +++ b/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/PullHttpChangeIngestorSSLTest.java @@ -17,7 +17,7 @@ package org.apache.nifi.minifi.bootstrap.configuration.ingestors; -import org.apache.nifi.minifi.bootstrap.ConfigurationFileHolder; +import org.apache.nifi.c2.client.api.ConfigurationFileHolder; import org.apache.nifi.minifi.bootstrap.configuration.ingestors.common.PullHttpChangeIngestorCommonTest; import org.eclipse.jetty.server.ServerConnector; import org.eclipse.jetty.util.ssl.SslContextFactory; diff --git a/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/PullHttpChangeIngestorTest.java b/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/PullHttpChangeIngestorTest.java index a3f05a5b48..6227a1fdc1 100644 --- a/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/PullHttpChangeIngestorTest.java +++ b/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/PullHttpChangeIngestorTest.java @@ -17,7 +17,7 @@ package org.apache.nifi.minifi.bootstrap.configuration.ingestors; -import org.apache.nifi.minifi.bootstrap.ConfigurationFileHolder; +import org.apache.nifi.c2.client.api.ConfigurationFileHolder; import org.apache.nifi.minifi.bootstrap.configuration.ingestors.common.PullHttpChangeIngestorCommonTest; import org.eclipse.jetty.server.ServerConnector; import org.junit.jupiter.api.BeforeAll; diff --git a/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/RestChangeIngestorSSLTest.java b/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/RestChangeIngestorSSLTest.java index 8f3edd1a64..7687f898a7 100644 --- a/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/RestChangeIngestorSSLTest.java +++ b/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/RestChangeIngestorSSLTest.java @@ -18,7 +18,7 @@ package org.apache.nifi.minifi.bootstrap.configuration.ingestors; import okhttp3.OkHttpClient; -import org.apache.nifi.minifi.bootstrap.ConfigurationFileHolder; +import org.apache.nifi.c2.client.api.ConfigurationFileHolder; import org.apache.nifi.minifi.bootstrap.configuration.ConfigurationChangeListener; import org.apache.nifi.minifi.bootstrap.configuration.ConfigurationChangeNotifier; import org.apache.nifi.minifi.bootstrap.configuration.ListenerHandleResult; diff --git a/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/RestChangeIngestorTest.java b/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/RestChangeIngestorTest.java index cfa934f21c..a1203f09fd 100644 --- a/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/RestChangeIngestorTest.java +++ b/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/RestChangeIngestorTest.java @@ -18,7 +18,7 @@ package org.apache.nifi.minifi.bootstrap.configuration.ingestors; import okhttp3.OkHttpClient; -import org.apache.nifi.minifi.bootstrap.ConfigurationFileHolder; +import org.apache.nifi.c2.client.api.ConfigurationFileHolder; import org.apache.nifi.minifi.bootstrap.configuration.ConfigurationChangeNotifier; import org.apache.nifi.minifi.bootstrap.configuration.ingestors.common.RestChangeIngestorCommonTest; import org.junit.jupiter.api.AfterAll; diff --git a/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/common/PullHttpChangeIngestorCommonTest.java b/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/common/PullHttpChangeIngestorCommonTest.java index 1bb309dccc..13fc37bd2c 100644 --- a/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/common/PullHttpChangeIngestorCommonTest.java +++ b/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/common/PullHttpChangeIngestorCommonTest.java @@ -17,11 +17,11 @@ package org.apache.nifi.minifi.bootstrap.configuration.ingestors.common; +import org.apache.nifi.c2.client.api.Differentiator; import org.apache.nifi.minifi.bootstrap.RunMiNiFi; import org.apache.nifi.minifi.bootstrap.configuration.ConfigurationChangeListener; import org.apache.nifi.minifi.bootstrap.configuration.ConfigurationChangeNotifier; import org.apache.nifi.minifi.bootstrap.configuration.ListenerHandleResult; -import org.apache.nifi.minifi.bootstrap.configuration.differentiators.interfaces.Differentiator; import org.apache.nifi.minifi.bootstrap.configuration.ingestors.PullHttpChangeIngestor; import org.apache.nifi.minifi.bootstrap.util.ByteBufferInputStream; import org.apache.nifi.minifi.commons.schema.ConfigSchema; diff --git a/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/common/RestChangeIngestorCommonTest.java b/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/common/RestChangeIngestorCommonTest.java index 74d88d7c74..021fe7b2c3 100644 --- a/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/common/RestChangeIngestorCommonTest.java +++ b/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/common/RestChangeIngestorCommonTest.java @@ -23,10 +23,10 @@ import okhttp3.OkHttpClient; import okhttp3.Request; import okhttp3.RequestBody; import okhttp3.Response; +import org.apache.nifi.c2.client.api.Differentiator; import org.apache.nifi.minifi.bootstrap.configuration.ConfigurationChangeListener; import org.apache.nifi.minifi.bootstrap.configuration.ConfigurationChangeNotifier; import org.apache.nifi.minifi.bootstrap.configuration.ListenerHandleResult; -import org.apache.nifi.minifi.bootstrap.configuration.differentiators.interfaces.Differentiator; import org.apache.nifi.minifi.bootstrap.configuration.ingestors.RestChangeIngestor; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; diff --git a/minifi/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/common/BootstrapPropertyKeys.java b/minifi/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/common/BootstrapPropertyKeys.java index 5a1becaa73..126f72dbec 100644 --- a/minifi/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/common/BootstrapPropertyKeys.java +++ b/minifi/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/common/BootstrapPropertyKeys.java @@ -17,9 +17,11 @@ package org.apache.nifi.minifi.commons.schema.common; -import org.apache.nifi.minifi.commons.schema.ProvenanceReportingSchema; -import org.apache.nifi.minifi.commons.schema.SecurityPropertiesSchema; -import org.apache.nifi.minifi.commons.schema.SensitivePropsSchema; +import static org.apache.nifi.minifi.commons.schema.RemoteProcessGroupSchema.TIMEOUT_KEY; +import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.COMMENT_KEY; +import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.SCHEDULING_PERIOD_KEY; +import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.SCHEDULING_STRATEGY_KEY; +import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.USE_COMPRESSION_KEY; import java.util.Arrays; import java.util.Collections; @@ -27,18 +29,12 @@ import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.Set; - -import static org.apache.nifi.minifi.commons.schema.RemoteProcessGroupSchema.TIMEOUT_KEY; -import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.COMMENT_KEY; -import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.SCHEDULING_PERIOD_KEY; -import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.SCHEDULING_STRATEGY_KEY; -import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.USE_COMPRESSION_KEY; +import org.apache.nifi.minifi.commons.schema.ProvenanceReportingSchema; +import org.apache.nifi.minifi.commons.schema.SecurityPropertiesSchema; +import org.apache.nifi.minifi.commons.schema.SensitivePropsSchema; public class BootstrapPropertyKeys { - public static final String NOTIFIER_PROPERTY_PREFIX = "nifi.minifi.notifier"; - public static final String NOTIFIER_COMPONENTS_KEY = NOTIFIER_PROPERTY_PREFIX + ".components"; - public static final String STATUS_REPORTER_PROPERTY_PREFIX = "nifi.minifi.status.reporter"; public static final String STATUS_REPORTER_COMPONENTS_KEY = STATUS_REPORTER_PROPERTY_PREFIX + ".components"; diff --git a/minifi/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/serialization/SchemaLoader.java b/minifi/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/serialization/SchemaLoader.java index d3eb766b5f..6be8b564d3 100644 --- a/minifi/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/serialization/SchemaLoader.java +++ b/minifi/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/serialization/SchemaLoader.java @@ -17,12 +17,14 @@ package org.apache.nifi.minifi.commons.schema.serialization; +import java.io.Writer; import org.apache.nifi.minifi.commons.schema.ConfigSchema; import org.apache.nifi.minifi.commons.schema.common.ConvertableSchema; import org.apache.nifi.minifi.commons.schema.common.StringUtil; import org.apache.nifi.minifi.commons.schema.exception.SchemaLoaderException; import org.apache.nifi.minifi.commons.schema.v1.ConfigSchemaV1; import org.apache.nifi.minifi.commons.schema.v2.ConfigSchemaV2; +import org.yaml.snakeyaml.DumperOptions; import org.yaml.snakeyaml.Yaml; import org.yaml.snakeyaml.error.YAMLException; @@ -67,6 +69,15 @@ public class SchemaLoader { } } + public static void toYaml(ConfigSchema schema, Writer writer) { + final DumperOptions options = new DumperOptions(); + options.setDefaultFlowStyle(DumperOptions.FlowStyle.BLOCK); + options.setPrettyFlow(true); + + Yaml yaml = new Yaml(options); + yaml.dump(schema.toMap(), writer); + } + public static ConfigSchema loadConfigSchemaFromYaml(InputStream sourceStream) throws IOException, SchemaLoaderException { return loadConfigSchemaFromYaml(loadYamlAsMap(sourceStream)); } diff --git a/minifi/minifi-docker/dockermaven/Dockerfile b/minifi/minifi-docker/dockermaven/Dockerfile index 9f0491cdc3..0b384441d4 100644 --- a/minifi/minifi-docker/dockermaven/Dockerfile +++ b/minifi/minifi-docker/dockermaven/Dockerfile @@ -16,7 +16,9 @@ # under the License. # -FROM openjdk:8-jre-alpine +ARG IMAGE_NAME=openjdk +ARG IMAGE_TAG=8-jre-alpine +FROM ${IMAGE_NAME}:${IMAGE_TAG} MAINTAINER Apache MiNiFi # Values are set by Maven diff --git a/minifi/minifi-docker/pom.xml b/minifi/minifi-docker/pom.xml index cf6efddc54..bcbe0de0c2 100644 --- a/minifi/minifi-docker/pom.xml +++ b/minifi/minifi-docker/pom.xml @@ -30,6 +30,8 @@ limitations under the License. ${project.version} + openjdk + 8-jre-alpine @@ -57,6 +59,8 @@ limitations under the License. ${minifi.version} + ${docker.image.name} + ${docker.image.tag} 1000 1000 ${minifi.version} diff --git a/minifi/minifi-docs/src/main/markdown/minifi-java-agent-quick-start.md b/minifi/minifi-docs/src/main/markdown/minifi-java-agent-quick-start.md index 623115b01a..602686f300 100644 --- a/minifi/minifi-docs/src/main/markdown/minifi-java-agent-quick-start.md +++ b/minifi/minifi-docs/src/main/markdown/minifi-java-agent-quick-start.md @@ -87,10 +87,12 @@ For Windows users, navigate to the folder where MiNiFi was installed. Navigate t This launches MiNiFi and leaves it running in the foreground. To shut down NiFi, select the window that was launched and hold the Ctrl key while pressing C. -# Working with dataflows +# Working with DataFlows When you are working with a MiNiFi dataflow, you should design it, add any additional configuration your environment or use case requires, and then deploy your dataflow. MiNiFi is not designed to accommodate substantial mid-dataflow configuration. -## Setting up Your Dataflow +## Setting up Your DataFlow + +### Manually from a NiFi Dataflow You can use the MiNiFi Toolkit, located in your MiNiFi installation directory, and any NiFi instance to set up the dataflow you want MiNiFi to run: 1. Launch NiFi @@ -106,11 +108,51 @@ config.sh transform input_file output_file **Note:** You can use one template at a time, per MiNiFi instance. - **Result:** Once you have your _config.yml_ file in the `minifi/conf` directory, launch that instance of MiNiFi and your dataflow begins automatically. +### Utilizing a C2 Server via the c2 protocol +If you have a [C2 server](../../../../minifi-c2/README.md) running, you can expose the whole _config.yml_ for the agent to download. As the agent is heartbeating via the C2 protocol, changes in flow version will trigger automatic config updates. + +1. Launch C2 server +2. Configure MiNiFi for C2 capability +``` +c2.enable=true +c2.config.directory=./conf +c2.runtime.manifest.identifier=minifi +c2.runtime.type=minifi-java +c2.rest.url=http://localhost:10090/c2/config/heartbeat +c2.rest.url.ack=http://localhost:10090/c2/config/acknowledge +c2.agent.heartbeat.period=5000 +#(Optional) c2.rest.callTimeout=10 sec +#(Optional) c2.agent.identifier=123-456-789 +c2.agent.class=agentClassName +``` +3. Configure MiNiFi to recognize _config.yml_ changes +``` +nifi.minifi.notifier.ingestors=org.apache.nifi.minifi.bootstrap.configuration.ingestors.FileChangeIngestor +nifi.minifi.notifier.ingestors.file.config.path=./conf/config-new.yml +nifi.minifi.notifier.ingestors.file.polling.period.seconds=5 +``` +4. Start MiNiFi +5. When a new flow is available on the C2 server, MiNiFi will download it via C2 and restart itself to pick up the changes + +**Note:** Flow definitions are class based. Each class has one flow defined for it. As a result, all the agents belonging to the same class will get the flow at update. + +## Loading a New Dataflow + +### Manually +To load a new dataflow for a MiNiFi instance to run: + +1. Create a new _config.yml_ file with the new dataflow. +2. Replace the existing _config.yml_ in `minifi/conf` with the new file. +3. Restart MiNiFi. + +### Utilizing C2 protocol +1. Change the flow definition on the C2 Server +2. When a new flow is available on the C2 server, MiNiFi will download it via C2 and restart itself to pick up the changes + ## Using Processors Not Packaged with MiNiFi -MiNiFi is able to use following processors out of the box: +MiNiFi is able to use the following processors out of the box: * UpdateAttribute * AttributesToJSON * Base64EncodeContent @@ -272,13 +314,6 @@ minifi.sh flowStatus processor:TailFile:health,stats,bulletins For details on the `flowStatus` option, see the "FlowStatus Query Option" section of the [Administration Guide](https://nifi.apache.org/minifi/system-admin-guide.html). -## Loading a New Dataflow -You can load a new dataflow for a MiNiFi instance to run: - -1. Create a new _config.yml_ file with the new dataflow. -2. Replace the existing _config.yml_ in `minifi/conf` with the new file. -3. Restart MiNiFi. - ## Stopping MiNiFi You can stop MiNiFi at any time. diff --git a/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-resources/src/main/resources/bin/minifi.sh b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-resources/src/main/resources/bin/minifi.sh index 11446f3d37..8074dfe93b 100755 --- a/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-resources/src/main/resources/bin/minifi.sh +++ b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-resources/src/main/resources/bin/minifi.sh @@ -39,7 +39,7 @@ done # Compute the canonicalized name by finding the physical path # for the directory we're in and appending the target file. -PHYS_DIR=`pwd -P` +PHYS_DIR=$(pwd -P) SCRIPT_DIR=$PHYS_DIR SCRIPT_NAME=$(basename "$0") @@ -82,6 +82,11 @@ detectOS() { export LDR_CNTRL=MAXDATA=0xB0000000@DSA echo ${LDR_CNTRL} fi + # In addition to those, go around the linux space and query the widely + # adopted /etc/os-release to detect linux variants + if [ -f /etc/os-release ]; then + . /etc/os-release + fi } unlimitFD() { @@ -223,11 +228,26 @@ SERVICEDESCRIPTOR # Provide the user execute access on the file chmod u+x ${SVC_FILE} - rm -f "/etc/rc2.d/S65${SVC_NAME}" - ln -s "/etc/init.d/${SVC_NAME}" "/etc/rc2.d/S65${SVC_NAME}" || { echo "Could not create link /etc/rc2.d/S65${SVC_NAME}"; exit 1; } - rm -f "/etc/rc2.d/K65${SVC_NAME}" - ln -s "/etc/init.d/${SVC_NAME}" "/etc/rc2.d/K65${SVC_NAME}" || { echo "Could not create link /etc/rc2.d/K65${SVC_NAME}"; exit 1; } - echo "Service ${SVC_NAME} installed" + # If SLES or OpenSuse... + if [ "${ID}" = "opensuse" ] || [ "${ID}" = "sles" ]; then + rm -f "/etc/rc.d/rc2.d/S65${SVC_NAME}" + ln -s "/etc/init.d/${SVC_NAME}" "/etc/rc.d/rc2.d/S65${SVC_NAME}" || { echo "Could not create link /etc/rc.d/rc2.d/S65${SVC_NAME}"; exit 1; } + rm -f "/etc/rc.d/rc2.d/K65${SVC_NAME}" + ln -s "/etc/init.d/${SVC_NAME}" "/etc/rc.d/rc2.d/K65${SVC_NAME}" || { echo "Could not create link /etc/rc.d/rc2.d/K65${SVC_NAME}"; exit 1; } + echo "Service ${SVC_NAME} installed" + # Anything other fallback to the old approach + else + rm -f "/etc/rc2.d/S65${SVC_NAME}" + ln -s "/etc/init.d/${SVC_NAME}" "/etc/rc2.d/S65${SVC_NAME}" || { echo "Could not create link /etc/rc2.d/S65${SVC_NAME}"; exit 1; } + rm -f "/etc/rc2.d/K65${SVC_NAME}" + ln -s "/etc/init.d/${SVC_NAME}" "/etc/rc2.d/K65${SVC_NAME}" || { echo "Could not create link /etc/rc2.d/K65${SVC_NAME}"; exit 1; } + echo "Service ${SVC_NAME} installed" + fi + + # systemd: generate minifi.service from init.d + if [ -d "/run/systemd/system/" ] || [ ! -z "$(pidof systemd 2>/dev/null)" ]; then + systemctl daemon-reload + fi } run() { @@ -236,15 +256,14 @@ run() { MINIFI_LIBS="${MINIFI_HOME}/lib/*" BOOTSTRAP_LIBS="${MINIFI_HOME}/lib/bootstrap/*" - run_as=$(grep '^\s*run.as' "${BOOTSTRAP_CONF}" | cut -d'=' -f2) + run_as_user=$(grep '^\s*run.as' "${BOOTSTRAP_CONF}" | cut -d'=' -f2) # If the run as user is the same as that starting the process, ignore this configuration - if [ "$run_as" = "$(whoami)" ]; then - unset run_as + if [ "$run_as_user" = "$(whoami)" ]; then + unset run_as_user fi - sudo_cmd_prefix="" if $cygwin; then - if [ -n "${run_as}" ]; then + if [ -n "${run_as_user}" ]; then echo "The run.as option is not supported in a Cygwin environment. Exiting." exit 1 fi; @@ -262,11 +281,9 @@ run() { BOOTSTRAP_CLASSPATH="${TOOLS_JAR};${BOOTSTRAP_CLASSPATH};${MINIFI_LIBS}" fi else - if [ -n "${run_as}" ]; then - if id -u "${run_as}" >/dev/null 2>&1; then - sudo_cmd_prefix="sudo -u ${run_as}" - else - echo "The specified run.as user ${run_as} does not exist. Exiting." + if [ -n "${run_as_user}" ]; then + if ! id -u "${run_as_user}" >/dev/null 2>&1; then + echo "The specified run.as user ${run_as_user} does not exist. Exiting." exit 1 fi fi; @@ -290,16 +307,35 @@ run() { BOOTSTRAP_PID_PARAMS="-Dorg.apache.nifi.minifi.bootstrap.config.pid.dir="\""${MINIFI_PID_DIR}"\""" BOOTSTRAP_CONF_PARAMS="-Dorg.apache.nifi.minifi.bootstrap.config.file="\""${BOOTSTRAP_CONF}"\""" + # uncomment to allow debugging of the bootstrap process + #BOOTSTRAP_DEBUG_PARAMS="-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=8000" + BOOTSTRAP_DIR_PARAMS="${BOOTSTRAP_LOG_PARAMS} ${BOOTSTRAP_PID_PARAMS} ${BOOTSTRAP_CONF_PARAMS}" - RUN_MINIFI_CMD="cd "\""${MINIFI_HOME}"\"" && ${sudo_cmd_prefix} "\""${JAVA}"\"" -cp "\""${BOOTSTRAP_CLASSPATH}"\"" -Xms12m -Xmx24m ${BOOTSTRAP_DIR_PARAMS} org.apache.nifi.minifi.bootstrap.RunMiNiFi" + RUN_BOOTSTRAP_CMD="'${JAVA}' -cp '${BOOTSTRAP_CLASSPATH}' -Xms12m -Xmx24m ${BOOTSTRAP_DIR_PARAMS} ${BOOTSTRAP_DEBUG_PARAMS} org.apache.nifi.minifi.bootstrap.RunMiNiFi" + RUN_MINIFI_CMD="${RUN_BOOTSTRAP_CMD} $@" + + if [ -n "${run_as_user}" ]; then + preserve_environment=$(grep '^\s*preserve.environment' "${BOOTSTRAP_CONF}" | cut -d'=' -f2 | tr '[:upper:]' '[:lower:]') + SUDO="sudo" + if [ "$preserve_environment" = "true" ]; then + SUDO="sudo -E" + fi + # Provide SCRIPT_DIR and execute nifi-env for the run.as user command + RUN_MINIFI_CMD="${SUDO} -u ${run_as_user} sh -c \"SCRIPT_DIR='${SCRIPT_DIR}' && . '${SCRIPT_DIR}/minifi-env.sh' && ${RUN_MINIFI_CMD}\"" + fi + + if [ "$1" = "run" ]; then + # Use exec to handover PID to RunMiNiFi java process, instead of foking it as a child process + RUN_MINIFI_CMD="exec ${RUN_MINIFI_CMD}" + fi # run 'start' in the background because the process will continue to run, monitoring MiNiFi. # all other commands will terminate quickly so want to just wait for them if [ "$1" = "start" ]; then - (eval $RUN_MINIFI_CMD $@ &) + (eval "cd ${MINIFI_HOME} && ${RUN_MINIFI_CMD}" &) else - (eval $RUN_MINIFI_CMD $@) + eval "cd ${MINIFI_HOME} && ${RUN_MINIFI_CMD}" fi EXIT_STATUS=$? @@ -320,16 +356,10 @@ case "$1" in install) install "$@" ;; - start|stop|run|status|flowStatus|dump|env) + start|stop|restart|run|status|flowStatus|dump|env) main "$@" exit $EXIT_STATUS ;; - restart) - init - run "stop" - run "start" - exit $EXIT_STATUS - ;; *) echo "Usage minifi {start|stop|run|restart|status|flowStatus|dump|install}" ;; diff --git a/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-resources/src/main/resources/conf/bootstrap.conf b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-resources/src/main/resources/conf/bootstrap.conf index 8c61989217..ac05d01286 100644 --- a/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-resources/src/main/resources/conf/bootstrap.conf +++ b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-resources/src/main/resources/conf/bootstrap.conf @@ -59,10 +59,9 @@ nifi.minifi.provenance.reporting.batch.size= nifi.minifi.provenance.reporting.communications.timeout= # Ignore all processor SSL controller services and use parent minifi SSL instead - nifi.minifi.flow.use.parent.ssl=false +nifi.minifi.flow.use.parent.ssl=false # Notifiers to use for the associated agent, comma separated list of class names -#nifi.minifi.notifier.ingestors=org.apache.nifi.minifi.bootstrap.configuration.ingestors.FileChangeIngestor #nifi.minifi.notifier.ingestors=org.apache.nifi.minifi.bootstrap.configuration.ingestors.RestChangeIngestor #nifi.minifi.notifier.ingestors=org.apache.nifi.minifi.bootstrap.configuration.ingestors.PullHttpChangeIngestor @@ -129,3 +128,35 @@ java.arg.7=-Djava.security.egd=file:/dev/urandom #Set headless mode by default java.arg.14=-Djava.awt.headless=true + +# MiNiFi Command & Control Configuration +# C2 Properties +# Enabling C2 Uncomment each of the following options +#c2.enable=true +## define protocol parameters +#c2.rest.url= +#c2.rest.url.ack= +## c2 timeouts +#c2.rest.connectionTimeout=5 sec +#c2.rest.readTimeout=5 sec +#c2.rest.callTimeout=10 sec +## heartbeat in milliseconds +#c2.agent.heartbeat.period=5000 +## define parameters about your agent +#c2.agent.class= +#c2.config.directory=./conf +#c2.runtime.manifest.identifier=minifi +#c2.runtime.type=minifi-java +# Optional. Defaults to a hardware based unique identifier +#c2.agent.identifier= +## Define TLS security properties for C2 communications +#c2.security.truststore.location= +#c2.security.truststore.password= +#c2.security.truststore.type=JKS +#c2.security.keystore.location= +#c2.security.keystore.password= +#c2.security.keystore.type=JKS +# The following ingestor configuration needs to be enabled in order to apply configuration updates coming from C2 server +#nifi.minifi.notifier.ingestors=org.apache.nifi.minifi.bootstrap.configuration.ingestors.FileChangeIngestor +#nifi.minifi.notifier.ingestors.file.config.path=./conf/config-new.yml +#nifi.minifi.notifier.ingestors.file.polling.period.seconds=5 \ No newline at end of file diff --git a/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-runtime/pom.xml b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-runtime/pom.xml index dc6e330882..9026db3480 100644 --- a/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-runtime/pom.xml +++ b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-runtime/pom.xml @@ -62,5 +62,14 @@ limitations under the License. org.slf4j jul-to-slf4j + + + com.fasterxml.jackson.core + jackson-core + + + com.fasterxml.jackson.core + jackson-databind + diff --git a/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-runtime/src/main/java/org/apache/nifi/minifi/BootstrapListener.java b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-runtime/src/main/java/org/apache/nifi/minifi/BootstrapListener.java index e8c3c8449a..3466ea26c1 100644 --- a/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-runtime/src/main/java/org/apache/nifi/minifi/BootstrapListener.java +++ b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-runtime/src/main/java/org/apache/nifi/minifi/BootstrapListener.java @@ -16,12 +16,13 @@ */ package org.apache.nifi.minifi; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.databind.ObjectMapper; import java.io.BufferedReader; import java.io.BufferedWriter; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; -import java.io.ObjectOutputStream; import java.io.OutputStream; import java.io.OutputStreamWriter; import java.lang.management.LockInfo; @@ -41,7 +42,6 @@ import java.util.List; import java.util.UUID; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; - import org.apache.nifi.minifi.commons.status.FlowStatusReport; import org.apache.nifi.minifi.status.StatusRequestException; import org.apache.nifi.util.LimitingInputStream; @@ -55,6 +55,7 @@ public class BootstrapListener { private final MiNiFi minifi; private final int bootstrapPort; private final String secretKey; + private final ObjectMapper objectMapper; private volatile Listener listener; @@ -62,6 +63,9 @@ public class BootstrapListener { this.minifi = minifi; this.bootstrapPort = bootstrapPort; secretKey = UUID.randomUUID().toString(); + + objectMapper = new ObjectMapper(); + objectMapper.setSerializationInclusion(JsonInclude.Include.NON_NULL); } public void start() throws IOException { @@ -212,6 +216,10 @@ public class BootstrapListener { String flowStatusRequestString = request.getArgs()[0]; writeStatusReport(flowStatusRequestString, socket.getOutputStream()); break; + case ENV: + logger.info("Received ENV request from Bootstrap"); + writeEnv(socket.getOutputStream()); + break; } } catch (final Throwable t) { logger.error("Failed to process request from Bootstrap due to " + t.toString(), t); @@ -231,10 +239,22 @@ public class BootstrapListener { } private void writeStatusReport(String flowStatusRequestString, final OutputStream out) throws IOException, StatusRequestException { - ObjectOutputStream oos = new ObjectOutputStream(out); FlowStatusReport flowStatusReport = minifi.getMinifiServer().getStatusReport(flowStatusRequestString); - oos.writeObject(flowStatusReport); - oos.close(); + objectMapper.writeValue(out, flowStatusReport); + } + + private static void writeEnv(OutputStream out) throws IOException { + try (BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(out))) { + StringBuilder sb = new StringBuilder(); + + System.getProperties() + .entrySet() + .stream() + .forEach(entry -> sb.append(entry.getKey()).append("=").append(entry.getValue()).append("\n")); + + writer.write(sb.toString()); + writer.flush(); + } } private static void writeDump(final OutputStream out) throws IOException { @@ -394,7 +414,8 @@ public class BootstrapListener { SHUTDOWN, DUMP, PING, - FLOW_STATUS_REPORT + FLOW_STATUS_REPORT, + ENV } private final RequestType requestType; diff --git a/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-runtime/src/main/java/org/apache/nifi/minifi/MiNiFi.java b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-runtime/src/main/java/org/apache/nifi/minifi/MiNiFi.java index 3517f83ff8..c83d008b72 100644 --- a/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-runtime/src/main/java/org/apache/nifi/minifi/MiNiFi.java +++ b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-runtime/src/main/java/org/apache/nifi/minifi/MiNiFi.java @@ -45,7 +45,6 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; - public class MiNiFi { private static final Logger logger = LoggerFactory.getLogger(MiNiFi.class); @@ -73,7 +72,7 @@ public class MiNiFi { final File kerberosConfigFile = properties.getKerberosConfigurationFile(); if (kerberosConfigFile != null) { final String kerberosConfigFilePath = kerberosConfigFile.getAbsolutePath(); - logger.info("Setting java.security.krb5.conf to {}", new Object[]{kerberosConfigFilePath}); + logger.info("Setting java.security.krb5.conf to {}", kerberosConfigFilePath); System.setProperty("java.security.krb5.conf", kerberosConfigFilePath); } @@ -151,6 +150,7 @@ public class MiNiFi { } minifiServer = (MiNiFiServer) nifiServer; Thread.currentThread().setContextClassLoader(minifiServer.getClass().getClassLoader()); + // Filter out the framework NAR from being loaded by the NiFiServer minifiServer.initialize(properties, systemBundle, @@ -261,7 +261,7 @@ public class MiNiFi { public static void main(String[] args) { logger.info("Launching MiNiFi..."); try { - NiFiProperties niFiProperties = NiFiProperties.createBasicNiFiProperties(null, (Map) null); + NiFiProperties niFiProperties = NiFiProperties.createBasicNiFiProperties(null, (Map) null); new MiNiFi(niFiProperties); } catch (final Throwable t) { logger.error("Failure to launch MiNiFi due to " + t, t); diff --git a/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-server/src/main/java/org/apache/nifi/minifi/StandardMiNiFiServer.java b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-server/src/main/java/org/apache/nifi/minifi/StandardMiNiFiServer.java index 2e99c06014..4cd491691c 100644 --- a/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-server/src/main/java/org/apache/nifi/minifi/StandardMiNiFiServer.java +++ b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-server/src/main/java/org/apache/nifi/minifi/StandardMiNiFiServer.java @@ -37,5 +37,4 @@ public class StandardMiNiFiServer extends HeadlessNiFiServer implements MiNiFiSe public FlowStatusReport getStatusReport(String requestString) throws StatusRequestException { return StatusConfigReporter.getStatus(this.flowController, requestString, logger); } - } diff --git a/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-server-nar/pom.xml b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-server-nar/pom.xml index 23dd1650d1..ddd507dafc 100644 --- a/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-server-nar/pom.xml +++ b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-server-nar/pom.xml @@ -43,6 +43,11 @@ nifi-headless-server 1.17.0-SNAPSHOT + + org.apache.commons + commons-lang3 + compile + org.apache.nifi diff --git a/minifi/minifi-nar-bundles/minifi-framework-bundle/pom.xml b/minifi/minifi-nar-bundles/minifi-framework-bundle/pom.xml index 3674f5b6f7..9067c559b8 100644 --- a/minifi/minifi-nar-bundles/minifi-framework-bundle/pom.xml +++ b/minifi/minifi-nar-bundles/minifi-framework-bundle/pom.xml @@ -39,6 +39,11 @@ limitations under the License. minifi-framework-api 1.17.0-SNAPSHOT + + org.apache.nifi + c2-client-api + 1.17.0-SNAPSHOT + org.apache.nifi nifi-headless-server diff --git a/minifi/pom.xml b/minifi/pom.xml index 583fc40f4f..35c660dc50 100644 --- a/minifi/pom.xml +++ b/minifi/pom.xml @@ -448,11 +448,18 @@ limitations under the License. 1.17.0-SNAPSHOT + - com.squareup.okhttp3 - okhttp - 3.12.3 + org.apache.nifi + c2-client-api + 1.17.0-SNAPSHOT + + org.apache.nifi + c2-client-base + 1.17.0-SNAPSHOT + + org.apache.commons commons-compress diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/exception/ProcessorInstantiationException.java b/nifi-framework-api/src/main/java/org/apache/nifi/controller/exception/ProcessorInstantiationException.java similarity index 100% rename from nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/exception/ProcessorInstantiationException.java rename to nifi-framework-api/src/main/java/org/apache/nifi/controller/exception/ProcessorInstantiationException.java diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/diagnostics/GarbageCollection.java b/nifi-framework-api/src/main/java/org/apache/nifi/diagnostics/GarbageCollection.java similarity index 100% rename from nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/diagnostics/GarbageCollection.java rename to nifi-framework-api/src/main/java/org/apache/nifi/diagnostics/GarbageCollection.java diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/diagnostics/StorageUsage.java b/nifi-framework-api/src/main/java/org/apache/nifi/diagnostics/StorageUsage.java similarity index 96% rename from nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/diagnostics/StorageUsage.java rename to nifi-framework-api/src/main/java/org/apache/nifi/diagnostics/StorageUsage.java index 50e1741c00..d4a787d71f 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/diagnostics/StorageUsage.java +++ b/nifi-framework-api/src/main/java/org/apache/nifi/diagnostics/StorageUsage.java @@ -56,7 +56,7 @@ public class StorageUsage implements Cloneable { } public int getDiskUtilization() { - return DiagnosticUtils.getUtilization(getUsedSpace(), totalSpace); + return Math.round(((float)getUsedSpace() / totalSpace) * 100); } @Override diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/diagnostics/SystemDiagnostics.java b/nifi-framework-api/src/main/java/org/apache/nifi/diagnostics/SystemDiagnostics.java similarity index 98% rename from nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/diagnostics/SystemDiagnostics.java rename to nifi-framework-api/src/main/java/org/apache/nifi/diagnostics/SystemDiagnostics.java index ac9f1f6ea5..9f89990b99 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/diagnostics/SystemDiagnostics.java +++ b/nifi-framework-api/src/main/java/org/apache/nifi/diagnostics/SystemDiagnostics.java @@ -168,7 +168,7 @@ public class SystemDiagnostics implements Cloneable { if (maxHeap == -1) { return -1; } else { - return DiagnosticUtils.getUtilization(usedHeap, maxHeap); + return Math.round(((float)usedHeap / maxHeap) * 100); } } @@ -176,7 +176,7 @@ public class SystemDiagnostics implements Cloneable { if (maxNonHeap == -1) { return -1; } else { - return DiagnosticUtils.getUtilization(usedNonHeap, maxNonHeap); + return Math.round(((float)usedNonHeap / maxNonHeap) * 100); } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/pom.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/pom.xml index a3aaa8cf27..b9c7b38628 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/pom.xml +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/pom.xml @@ -185,6 +185,11 @@ org.xerial.snappy snappy-java + + org.apache.nifi + c2-client-service + 1.17.0-SNAPSHOT + org.apache.zookeeper zookeeper diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/c2/C2NiFiProperties.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/c2/C2NiFiProperties.java new file mode 100644 index 0000000000..7983d9967a --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/c2/C2NiFiProperties.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.c2; + +import java.util.concurrent.TimeUnit; + +public class C2NiFiProperties { + + public static final String C2_PREFIX = "c2."; + + public static final String C2_ENABLE_KEY = C2_PREFIX + "enable"; + public static final String C2_AGENT_PROTOCOL_KEY = C2_PREFIX + "agent.protocol.class"; + public static final String C2_COAP_HOST_KEY = C2_PREFIX + "agent.coap.host"; + public static final String C2_COAP_PORT_KEY = C2_PREFIX + "agent.coap.port"; + public static final String C2_CONFIG_DIRECTORY_KEY = C2_PREFIX + "config.directory"; + public static final String C2_RUNTIME_MANIFEST_IDENTIFIER_KEY = C2_PREFIX + "runtime.manifest.identifier"; + public static final String C2_RUNTIME_TYPE_KEY = C2_PREFIX + "runtime.type"; + public static final String C2_REST_URL_KEY = C2_PREFIX + "rest.url"; + public static final String C2_REST_URL_ACK_KEY = C2_PREFIX + "rest.url.ack"; + public static final String C2_ROOT_CLASSES_KEY = C2_PREFIX + "root.classes"; + public static final String C2_AGENT_HEARTBEAT_PERIOD_KEY = C2_PREFIX + "agent.heartbeat.period"; + public static final String C2_CONNECTION_TIMEOUT = C2_PREFIX + "rest.connectionTimeout"; + public static final String C2_READ_TIMEOUT = C2_PREFIX + "rest.readTimeout"; + public static final String C2_CALL_TIMEOUT = C2_PREFIX + "rest.callTimeout"; + public static final String C2_AGENT_CLASS_KEY = C2_PREFIX + "agent.class"; + public static final String C2_AGENT_IDENTIFIER_KEY = C2_PREFIX + "agent.identifier"; + + public static final String C2_ROOT_CLASS_DEFINITIONS_KEY = C2_PREFIX + "root.class.definitions"; + public static final String C2_METRICS_NAME_KEY = C2_ROOT_CLASS_DEFINITIONS_KEY + ".metrics.name"; + public static final String C2_METRICS_METRICS_KEY = C2_ROOT_CLASS_DEFINITIONS_KEY + ".metrics.metrics"; + public static final String C2_METRICS_METRICS_TYPED_METRICS_NAME_KEY = C2_ROOT_CLASS_DEFINITIONS_KEY + ".metrics.metrics.typedmetrics.name"; + public static final String C2_METRICS_METRICS_QUEUED_METRICS_NAME_KEY = C2_ROOT_CLASS_DEFINITIONS_KEY + ".metrics.metrics.queuemetrics.name"; + public static final String C2_METRICS_METRICS_QUEUE_METRICS_CLASSES_KEY = C2_ROOT_CLASS_DEFINITIONS_KEY + ".metrics.metrics.queuemetrics.classes"; + public static final String C2_METRICS_METRICS_TYPED_METRICS_CLASSES_KEY = C2_ROOT_CLASS_DEFINITIONS_KEY + ".metrics.metrics.typedmetrics.classes"; + public static final String C2_METRICS_METRICS_PROCESSOR_METRICS_NAME_KEY = C2_ROOT_CLASS_DEFINITIONS_KEY + ".metrics.metrics.processorMetrics.name"; + public static final String C2_METRICS_METRICS_PROCESSOR_METRICS_CLASSES_KEY = C2_ROOT_CLASS_DEFINITIONS_KEY + ".metrics.metrics.processorMetrics.classes"; + + /* C2 Client Security Properties */ + private static final String C2_REST_SECURITY_BASE_KEY = C2_PREFIX + "security"; + public static final String TRUSTSTORE_LOCATION_KEY = C2_REST_SECURITY_BASE_KEY + ".truststore.location"; + public static final String TRUSTSTORE_PASSWORD_KEY = C2_REST_SECURITY_BASE_KEY + ".truststore.password"; + public static final String TRUSTSTORE_TYPE_KEY = C2_REST_SECURITY_BASE_KEY + ".truststore.type"; + public static final String KEYSTORE_LOCATION_KEY = C2_REST_SECURITY_BASE_KEY + ".keystore.location"; + public static final String KEYSTORE_PASSWORD_KEY = C2_REST_SECURITY_BASE_KEY + ".keystore.password"; + public static final String KEYSTORE_TYPE_KEY = C2_REST_SECURITY_BASE_KEY + ".keystore.type"; + + // Defaults + // Heartbeat period of 1 second + public static final long C2_AGENT_DEFAULT_HEARTBEAT_PERIOD = TimeUnit.SECONDS.toMillis(1); + + // Connection timeout of 5 seconds + public static final String C2_DEFAULT_CONNECTION_TIMEOUT = "5 sec"; + // Read timeout of 5 seconds + public static final String C2_DEFAULT_READ_TIMEOUT = "5 sec"; + // Call timeout of 10 seconds + public static final String C2_DEFAULT_CALL_TIMEOUT = "10 sec"; +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/c2/C2NifiClientService.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/c2/C2NifiClientService.java new file mode 100644 index 0000000000..39eef4e78c --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/c2/C2NifiClientService.java @@ -0,0 +1,211 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.c2; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Optional; +import org.apache.nifi.c2.client.C2ClientConfig; +import org.apache.nifi.c2.client.http.C2HttpClient; +import org.apache.nifi.c2.client.service.C2ClientService; +import org.apache.nifi.c2.client.service.C2HeartbeatFactory; +import org.apache.nifi.c2.client.service.FlowIdHolder; +import org.apache.nifi.c2.client.service.model.RuntimeInfoWrapper; +import org.apache.nifi.c2.protocol.api.AgentRepositories; +import org.apache.nifi.c2.protocol.api.AgentRepositoryStatus; +import org.apache.nifi.c2.protocol.api.FlowQueueStatus; +import org.apache.nifi.c2.serializer.C2JacksonSerializer; +import org.apache.nifi.controller.FlowController;; +import org.apache.nifi.controller.status.ConnectionStatus; +import org.apache.nifi.controller.status.ProcessGroupStatus; +import org.apache.nifi.diagnostics.StorageUsage; +import org.apache.nifi.diagnostics.SystemDiagnostics; +import org.apache.nifi.extension.manifest.parser.ExtensionManifestParser; +import org.apache.nifi.extension.manifest.parser.jaxb.JAXBExtensionManifestParser; +import org.apache.nifi.manifest.RuntimeManifestService; +import org.apache.nifi.manifest.StandardRuntimeManifestService; +import org.apache.nifi.nar.ExtensionManagerHolder; +import org.apache.nifi.services.FlowService; +import org.apache.nifi.util.FormatUtils; +import org.apache.nifi.util.NiFiProperties; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collection; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +public class C2NifiClientService { + + private static final Logger logger = LoggerFactory.getLogger(C2NifiClientService.class); + private static final String DEFAULT_CONF_DIR = "./conf"; + private static final String TARGET_CONFIG_FILE = "/config-new.yml"; + private static final String ROOT_GROUP_ID = "root"; + private static final Long INITIAL_DELAY = 10000L; + private static final Integer TERMINATION_WAIT = 5000; + + private final C2ClientService c2ClientService; + + private final FlowService flowService; + private final FlowController flowController; + private final String propertiesDir; + private final ScheduledThreadPoolExecutor scheduledExecutorService = new ScheduledThreadPoolExecutor(1); + private final ExtensionManifestParser extensionManifestParser = new JAXBExtensionManifestParser(); + + private final RuntimeManifestService runtimeManifestService; + private final long heartbeatPeriod; + + public C2NifiClientService(final NiFiProperties niFiProperties, final FlowService flowService, final FlowController flowController) { + C2ClientConfig clientConfig = generateClientConfig(niFiProperties); + FlowIdHolder flowIdHolder = new FlowIdHolder(clientConfig.getConfDirectory()); + this.propertiesDir = niFiProperties.getProperty(NiFiProperties.PROPERTIES_FILE_PATH, null); + this.runtimeManifestService = new StandardRuntimeManifestService( + ExtensionManagerHolder.getExtensionManager(), + extensionManifestParser, + clientConfig.getRuntimeManifestIdentifier(), + clientConfig.getRuntimeType() + ); + this.heartbeatPeriod = clientConfig.getHeartbeatPeriod(); + this.flowService = flowService; + this.flowController = flowController; + this.c2ClientService = new C2ClientService( + new C2HttpClient(clientConfig, new C2JacksonSerializer()), + new C2HeartbeatFactory(clientConfig, flowIdHolder), + flowIdHolder, + this::updateFlowContent + ); + } + + private C2ClientConfig generateClientConfig(NiFiProperties properties) { + return new C2ClientConfig.Builder() + .agentClass(properties.getProperty(C2NiFiProperties.C2_AGENT_CLASS_KEY, "")) + .agentIdentifier(properties.getProperty(C2NiFiProperties.C2_AGENT_IDENTIFIER_KEY)) + .heartbeatPeriod(Long.parseLong(properties.getProperty(C2NiFiProperties.C2_AGENT_HEARTBEAT_PERIOD_KEY, + String.valueOf(C2NiFiProperties.C2_AGENT_DEFAULT_HEARTBEAT_PERIOD)))) + .connectTimeout((long) FormatUtils.getPreciseTimeDuration(properties.getProperty(C2NiFiProperties.C2_CONNECTION_TIMEOUT, + C2NiFiProperties.C2_DEFAULT_CONNECTION_TIMEOUT), TimeUnit.MILLISECONDS)) + .readTimeout((long) FormatUtils.getPreciseTimeDuration(properties.getProperty(C2NiFiProperties.C2_READ_TIMEOUT, + C2NiFiProperties.C2_DEFAULT_READ_TIMEOUT), TimeUnit.MILLISECONDS)) + .callTimeout((long) FormatUtils.getPreciseTimeDuration(properties.getProperty(C2NiFiProperties.C2_CALL_TIMEOUT, + C2NiFiProperties.C2_DEFAULT_CALL_TIMEOUT), TimeUnit.MILLISECONDS)) + .c2Url(properties.getProperty(C2NiFiProperties.C2_REST_URL_KEY, "")) + .confDirectory(properties.getProperty(C2NiFiProperties.C2_CONFIG_DIRECTORY_KEY, DEFAULT_CONF_DIR)) + .runtimeManifestIdentifier(properties.getProperty(C2NiFiProperties.C2_RUNTIME_MANIFEST_IDENTIFIER_KEY, "")) + .runtimeType(properties.getProperty(C2NiFiProperties.C2_RUNTIME_TYPE_KEY, "")) + .c2AckUrl(properties.getProperty(C2NiFiProperties.C2_REST_URL_ACK_KEY, "")) + .truststoreFilename(properties.getProperty(C2NiFiProperties.TRUSTSTORE_LOCATION_KEY, "")) + .truststorePassword(properties.getProperty(C2NiFiProperties.TRUSTSTORE_PASSWORD_KEY, "")) + .truststoreType(properties.getProperty(C2NiFiProperties.TRUSTSTORE_TYPE_KEY, "JKS")) + .keystoreFilename(properties.getProperty(C2NiFiProperties.KEYSTORE_LOCATION_KEY, "")) + .keystorePassword(properties.getProperty(C2NiFiProperties.KEYSTORE_PASSWORD_KEY, "")) + .keystoreType(properties.getProperty(C2NiFiProperties.KEYSTORE_TYPE_KEY, "JKS")) + .build(); + } + + public void start() { + try { + scheduledExecutorService.scheduleAtFixedRate(() -> c2ClientService.sendHeartbeat(generateRuntimeInfo()), INITIAL_DELAY, heartbeatPeriod, TimeUnit.MILLISECONDS); + } catch (Exception e) { + logger.error("Could not start C2 Client Heartbeat Reporting", e); + throw new RuntimeException(e); + } + } + + public void stop() { + try { + scheduledExecutorService.shutdown(); + scheduledExecutorService.awaitTermination(TERMINATION_WAIT, TimeUnit.MILLISECONDS); + } catch (InterruptedException ignore) { + logger.info("Stopping C2 Client's thread was interrupted but shutting down anyway the C2NifiClientService"); + } + } + + private RuntimeInfoWrapper generateRuntimeInfo() { + return new RuntimeInfoWrapper(getAgentRepositories(), runtimeManifestService.getManifest(), getQueueStatus()); + } + + private AgentRepositories getAgentRepositories() { + final SystemDiagnostics systemDiagnostics = flowController.getSystemDiagnostics(); + + final AgentRepositories repos = new AgentRepositories(); + final AgentRepositoryStatus flowFileRepoStatus = new AgentRepositoryStatus(); + final StorageUsage ffRepoStorageUsage = systemDiagnostics.getFlowFileRepositoryStorageUsage(); + flowFileRepoStatus.setDataSize(ffRepoStorageUsage.getUsedSpace()); + flowFileRepoStatus.setDataSizeMax(ffRepoStorageUsage.getTotalSpace()); + repos.setFlowFile(flowFileRepoStatus); + + final AgentRepositoryStatus provRepoStatus = new AgentRepositoryStatus(); + final Iterator> provRepoStorageUsages = systemDiagnostics.getProvenanceRepositoryStorageUsage().entrySet().iterator(); + if (provRepoStorageUsages.hasNext()) { + final StorageUsage provRepoStorageUsage = provRepoStorageUsages.next().getValue(); + provRepoStatus.setDataSize(provRepoStorageUsage.getUsedSpace()); + provRepoStatus.setDataSizeMax(provRepoStorageUsage.getTotalSpace()); + } + + repos.setProvenance(provRepoStatus); + + return repos; + } + + private Map getQueueStatus() { + ProcessGroupStatus rootProcessGroupStatus = flowController.getEventAccess().getGroupStatus(ROOT_GROUP_ID); + + final Collection connectionStatuses = rootProcessGroupStatus.getConnectionStatus(); + + final Map processGroupStatus = new HashMap<>(); + for (ConnectionStatus connectionStatus : connectionStatuses) { + final FlowQueueStatus flowQueueStatus = new FlowQueueStatus(); + + flowQueueStatus.setSize((long) connectionStatus.getQueuedCount()); + flowQueueStatus.setSizeMax(connectionStatus.getBackPressureObjectThreshold()); + + flowQueueStatus.setDataSize(connectionStatus.getQueuedBytes()); + flowQueueStatus.setDataSizeMax(connectionStatus.getBackPressureBytesThreshold()); + + processGroupStatus.put(connectionStatus.getId(), flowQueueStatus); + } + + return processGroupStatus; + } + + private boolean updateFlowContent(byte[] updateContent) { + logger.debug("Update content: \n{}", new String(updateContent, StandardCharsets.UTF_8)); + Path path = getTargetConfigFile().toPath(); + try { + Files.write(getTargetConfigFile().toPath(), updateContent); + logger.info("Updated configuration was written to: {}", path); + return true; + } catch (IOException e) { + logger.error("Configuration update failed. File creation was not successful targeting: {}", path, e); + return false; + } + } + + private File getTargetConfigFile() { + return Optional.ofNullable(propertiesDir) + .map(File::new) + .map(File::getParent) + .map(parentDir -> new File(parentDir + TARGET_CONFIG_FILE)) + .orElse( new File(DEFAULT_CONF_DIR + TARGET_CONFIG_FILE)); + } +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardFlowService.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardFlowService.java index 16db9a07f7..c7ddf57a99 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardFlowService.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardFlowService.java @@ -21,6 +21,8 @@ import org.apache.nifi.authorization.Authorizer; import org.apache.nifi.authorization.AuthorizerCapabilityDetection; import org.apache.nifi.authorization.ManagedAuthorizer; import org.apache.nifi.bundle.Bundle; +import org.apache.nifi.c2.C2NiFiProperties; +import org.apache.nifi.c2.C2NifiClientService; import org.apache.nifi.cluster.ConnectionException; import org.apache.nifi.cluster.coordination.ClusterCoordinator; import org.apache.nifi.cluster.coordination.node.ClusterRoles; @@ -145,6 +147,9 @@ public class StandardFlowService implements FlowService, ProtocolHandler { */ private NodeIdentifier nodeId; + /* A reference to the client service for handling*/ + private C2NifiClientService c2NifiClientService; + // guardedBy rwLock private boolean firstControllerInitialization = true; @@ -290,6 +295,17 @@ public class StandardFlowService implements FlowService, ProtocolHandler { if (configuredForClustering) { senderListener.start(); + } else { + // If standalone and C2 is enabled, create a C2 client + final boolean c2Enabled = Boolean.parseBoolean(nifiProperties.getProperty(C2NiFiProperties.C2_ENABLE_KEY, "false")); + if (c2Enabled) { + logger.info("C2 enabled, creating a C2 client instance"); + c2NifiClientService = new C2NifiClientService(nifiProperties, this, this.controller); + c2NifiClientService.start(); + } else { + logger.info("C2 Property [{}] missing or disabled: C2 client not created", C2NiFiProperties.C2_ENABLE_KEY); + c2NifiClientService = null; + } } } catch (final IOException ioe) { @@ -315,6 +331,10 @@ public class StandardFlowService implements FlowService, ProtocolHandler { running.set(false); + if (c2NifiClientService != null) { + c2NifiClientService.stop(); + } + if (clusterCoordinator != null) { try { clusterCoordinator.shutdown(); diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/manifest/StandardRuntimeManifestService.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/manifest/StandardRuntimeManifestService.java index 74473a2d05..36fa80cb62 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/manifest/StandardRuntimeManifestService.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/manifest/StandardRuntimeManifestService.java @@ -47,10 +47,19 @@ public class StandardRuntimeManifestService implements RuntimeManifestService { private final ExtensionManager extensionManager; private final ExtensionManifestParser extensionManifestParser; + private final String runtimeManifestIdentifier; + private final String runtimeType; - public StandardRuntimeManifestService(final ExtensionManager extensionManager, final ExtensionManifestParser extensionManifestParser) { + public StandardRuntimeManifestService(final ExtensionManager extensionManager, final ExtensionManifestParser extensionManifestParser, + final String runtimeManifestIdentifier, final String runtimeType) { this.extensionManager = extensionManager; this.extensionManifestParser = extensionManifestParser; + this.runtimeManifestIdentifier = runtimeManifestIdentifier; + this.runtimeType = runtimeType; + } + + public StandardRuntimeManifestService(final ExtensionManager extensionManager, final ExtensionManifestParser extensionManifestParser) { + this(extensionManager, extensionManifestParser, RUNTIME_MANIFEST_IDENTIFIER, RUNTIME_TYPE); } @Override @@ -68,14 +77,14 @@ public class StandardRuntimeManifestService implements RuntimeManifestService { buildInfo.setTimestamp(frameworkBuildDate == null ? null : frameworkBuildDate.getTime()); final RuntimeManifestBuilder manifestBuilder = new StandardRuntimeManifestBuilder() - .identifier(RUNTIME_MANIFEST_IDENTIFIER) - .runtimeType(RUNTIME_TYPE) + .identifier(runtimeManifestIdentifier) + .runtimeType(runtimeType) .version(buildInfo.getVersion()) .schedulingDefaults(SchedulingDefaultsFactory.getNifiSchedulingDefaults()) .buildInfo(buildInfo); for (final Bundle bundle : allBundles) { - getExtensionManifest(bundle).ifPresent(em -> manifestBuilder.addBundle(em)); + getExtensionManifest(bundle).ifPresent(manifestBuilder::addBundle); } return manifestBuilder.build(); diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-headless-server/src/main/java/org/apache/nifi/headless/HeadlessNiFiServer.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-headless-server/src/main/java/org/apache/nifi/headless/HeadlessNiFiServer.java index 620b640af0..0d726eceb4 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-headless-server/src/main/java/org/apache/nifi/headless/HeadlessNiFiServer.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-headless-server/src/main/java/org/apache/nifi/headless/HeadlessNiFiServer.java @@ -28,6 +28,7 @@ import org.apache.nifi.authorization.exception.AuthorizationAccessException; import org.apache.nifi.authorization.exception.AuthorizerCreationException; import org.apache.nifi.authorization.exception.AuthorizerDestructionException; import org.apache.nifi.bundle.Bundle; +import org.apache.nifi.c2.client.api.C2Client; import org.apache.nifi.controller.DecommissionTask; import org.apache.nifi.controller.FlowController; import org.apache.nifi.controller.FlowSerializationStrategy; @@ -180,6 +181,7 @@ public class HeadlessNiFiServer implements NiFiServer { narAutoLoader = new NarAutoLoader(props, narLoader); narAutoLoader.start(); logger.info("Flow loaded successfully."); + } catch (Exception e) { // ensure the flow service is terminated if (flowService != null && flowService.isRunning()) { @@ -226,6 +228,10 @@ public class HeadlessNiFiServer implements NiFiServer { return null; } + protected C2Client getC2Client() { + return null; + } + public void stop() { try { flowService.stop(false); diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-nar-utils/pom.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-nar-utils/pom.xml index 15dd9e301b..11559ebfd0 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-nar-utils/pom.xml +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-nar-utils/pom.xml @@ -34,6 +34,11 @@ org.apache.nifi nifi-framework-api + + org.apache.nifi + c2-client-api + 1.17.0-SNAPSHOT + diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-nar-utils/src/main/java/org/apache/nifi/nar/NarClassLoaders.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-nar-utils/src/main/java/org/apache/nifi/nar/NarClassLoaders.java index 0f37ade095..f95ae8c4ef 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-nar-utils/src/main/java/org/apache/nifi/nar/NarClassLoaders.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-nar-utils/src/main/java/org/apache/nifi/nar/NarClassLoaders.java @@ -20,6 +20,7 @@ import org.apache.nifi.NiFiServer; import org.apache.nifi.bundle.Bundle; import org.apache.nifi.bundle.BundleCoordinate; import org.apache.nifi.bundle.BundleDetails; +import org.apache.nifi.c2.client.api.C2Client; import org.apache.nifi.util.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -40,7 +41,6 @@ import java.util.ServiceLoader; import java.util.Set; import java.util.stream.Collectors; - /** * Used to initialize the extension and framework classloaders. * @@ -69,6 +69,7 @@ public final class NarClassLoaders { final Bundle frameworkBundle, final Bundle jettyBundle, final NiFiServer serverInstance, + final C2Client c2ClientInstance, final Map bundles) { this.frameworkWorkingDir = frameworkDir; this.extensionWorkingDir = extensionDir; @@ -167,9 +168,10 @@ public final class NarClassLoaders { } NiFiServer serverInstance = null; + C2Client c2ClientInstance = null; if (!narWorkingDirContents.isEmpty()) { final List narDetails = new ArrayList<>(); - final Map narCoordinatesToWorkingDir = new HashMap<>(); + final Map narCoordinatesToWorkingDir = new HashMap<>(); // load the nar details which includes and nar dependencies for (final File unpackedNar : narWorkingDirContents) { @@ -177,8 +179,7 @@ public final class NarClassLoaders { try { narDetail = getNarDetails(unpackedNar); } catch (IllegalStateException e) { - logger.warn("Unable to load NAR {} due to {}, skipping...", - new Object[] {unpackedNar.getAbsolutePath(), e.getMessage()}); + logger.warn("Unable to load NAR {} due to {}, skipping...", unpackedNar.getAbsolutePath(), e.getMessage()); continue; } @@ -290,7 +291,7 @@ public final class NarClassLoaders { } else { Map.Entry nifiServer = niFiServers.entrySet().iterator().next(); serverInstance = nifiServer.getKey(); - logger.info("Found NiFiServer implementation {} in {}", new Object[]{serverInstance.getClass().getName(), nifiServer.getValue()}); + logger.info("Found NiFiServer implementation {} in {}", serverInstance.getClass().getName(), nifiServer.getValue()); } // see if any nars couldn't be loaded @@ -310,7 +311,7 @@ public final class NarClassLoaders { .filter(b -> b.getBundleDetails().getCoordinate().getId().equals(JETTY_NAR_ID)) .findFirst().orElse(null); - return new InitContext(frameworkWorkingDir, extensionsWorkingDir, frameworkBundle, jettyBundle, serverInstance, new LinkedHashMap<>(narDirectoryBundleLookup)); + return new InitContext(frameworkWorkingDir, extensionsWorkingDir, frameworkBundle, jettyBundle, serverInstance, c2ClientInstance, new LinkedHashMap<>(narDirectoryBundleLookup)); } /** @@ -365,7 +366,7 @@ public final class NarClassLoaders { initContext.bundles.put(bundleDetail.getWorkingDirectory().getCanonicalPath(), bundle); } } catch (final Exception e) { - logger.error("Unable to load NAR {} due to {}, skipping...", new Object[]{bundleDetail.getWorkingDirectory(), e.getMessage()}); + logger.error("Unable to load NAR {} due to {}, skipping...", bundleDetail.getWorkingDirectory(), e.getMessage()); } } @@ -383,7 +384,7 @@ public final class NarClassLoaders { return new NarLoadResult(loadedBundles, skippedBundles); } - private ClassLoader createBundleClassLoader(final BundleDetails bundleDetail, final Map> bundleIdToCoordinatesLookup, final boolean logDetails) + private ClassLoader createBundleClassLoader(final BundleDetails bundleDetail, final Map> bundleIdToCoordinatesLookup, final boolean logDetails) throws IOException, ClassNotFoundException { ClassLoader bundleClassLoader = null; @@ -457,7 +458,7 @@ public final class NarClassLoaders { } } catch (Exception e) { - logger.error("Unable to load NAR {} due to {}, skipping...", new Object[]{unpackedNar.getAbsolutePath(), e.getMessage()}); + logger.error("Unable to load NAR {} due to {}, skipping...", unpackedNar.getAbsolutePath(), e.getMessage()); } } return narDetails; @@ -536,7 +537,7 @@ public final class NarClassLoaders { } /** - * @return the Server class Bundle (NiFi Web/UI or MiNiFi) + * @return the Server class implementation (NiFi Web/UI or MiNiFi, e.g.) * * @throws IllegalStateException if the server Bundle has not been loaded */ @@ -562,7 +563,7 @@ public final class NarClassLoaders { try { return initContext.bundles.get(extensionWorkingDirectory.getCanonicalPath()); } catch (final IOException ioe) { - if(logger.isDebugEnabled()){ + if(logger.isDebugEnabled()) { logger.debug("Unable to get extension classloader for working directory '{}'", extensionWorkingDirectory); } return null; diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-jetty/src/main/java/org/apache/nifi/web/server/JettyServer.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-jetty/src/main/java/org/apache/nifi/web/server/JettyServer.java index 199b10e890..7063a7ecdc 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-jetty/src/main/java/org/apache/nifi/web/server/JettyServer.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-jetty/src/main/java/org/apache/nifi/web/server/JettyServer.java @@ -1057,7 +1057,7 @@ public class JettyServer implements NiFiServer, ExtensionUiLoader { } @Override - public void initialize(NiFiProperties properties, Bundle systemBundle, Set bundles, ExtensionMapping extensionMapping) { + public void initialize(final NiFiProperties properties, final Bundle systemBundle, final Set bundles, final ExtensionMapping extensionMapping) { this.props = properties; this.systemBundle = systemBundle; this.bundles = bundles; diff --git a/nifi-server-api/src/main/java/org/apache/nifi/NiFiServer.java b/nifi-server-api/src/main/java/org/apache/nifi/NiFiServer.java index 70a0f5e077..824fa83abb 100644 --- a/nifi-server-api/src/main/java/org/apache/nifi/NiFiServer.java +++ b/nifi-server-api/src/main/java/org/apache/nifi/NiFiServer.java @@ -26,7 +26,7 @@ import org.apache.nifi.util.NiFiProperties; import java.util.Set; /** - * + * The main interface for declaring a NiFi-based server application */ public interface NiFiServer {