Apache
/
nifi
mirror of https://github.com/apache/nifi.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

NIFI-11230 Added Runtime Configuration validation to nifi-bootstrap 

This closes #7196

Co-authored-by: David Handermann <exceptionfactory@apache.org>
Signed-off-by: David Handermann <exceptionfactory@apache.org>
This commit is contained in:
Emilio Setiadarma 2023-04-09 20:17:08 -07:00 committed by exceptionfactory
parent 73ecb54c42
commit 436fd91a25
No known key found for this signature in database
GPG Key ID: 29B6A52D2AAE8DBA
13 changed files with 891 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/RunNiFi.java
View File

@ -18,6 +18,7 @@ package org.apache.nifi.bootstrap;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.apache.nifi.bootstrap.notification.NotificationType; import org.apache.nifi.bootstrap.notification.NotificationType;
import org.apache.nifi.bootstrap.process.RuntimeValidatorExecutor;
import org.apache.nifi.bootstrap.util.DumpFileValidator; import org.apache.nifi.bootstrap.util.DumpFileValidator;
import org.apache.nifi.bootstrap.util.SecureNiFiConfigUtil; import org.apache.nifi.bootstrap.util.SecureNiFiConfigUtil;
import org.apache.nifi.util.file.FileUtils; import org.apache.nifi.util.file.FileUtils;
@ -146,6 +147,7 @@ public class RunNiFi {
private final Logger defaultLogger = LoggerFactory.getLogger(RunNiFi.class); private final Logger defaultLogger = LoggerFactory.getLogger(RunNiFi.class);
private final ExecutorService loggingExecutor; private final ExecutorService loggingExecutor;
private final RuntimeValidatorExecutor runtimeValidatorExecutor;
private volatile Set<Future<?>> loggingFutures = new HashSet<>(2); private volatile Set<Future<?>> loggingFutures = new HashSet<>(2);
private final NotificationServiceManager serviceManager; private final NotificationServiceManager serviceManager;
@ -163,6 +165,8 @@ public class RunNiFi {
}); });
serviceManager = loadServices(); serviceManager = loadServices();
runtimeValidatorExecutor = new RuntimeValidatorExecutor();
} }
private static void printUsage() { private static void printUsage() {
@ -1113,6 +1117,8 @@ public class RunNiFi {
cmdLogger.warn("Failed to delete previous lock file {}; this file should be cleaned up manually", prevLockFile); cmdLogger.warn("Failed to delete previous lock file {}; this file should be cleaned up manually", prevLockFile);
} }
runtimeValidatorExecutor.execute();
final ProcessBuilder builder = new ProcessBuilder(); final ProcessBuilder builder = new ProcessBuilder();
if (!bootstrapConfigFile.exists()) { if (!bootstrapConfigFile.exists()) {

106
nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/process/AbstractFileBasedRuntimeValidator.java Normal file
View File

@ -0,0 +1,106 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.bootstrap.process;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Scanner;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public abstract class AbstractFileBasedRuntimeValidator implements RuntimeValidator {
private final File configurationFile;
AbstractFileBasedRuntimeValidator(final File configurationFile) {
this.configurationFile = configurationFile;
}
@Override
public List<RuntimeValidatorResult> validate() {
final List<RuntimeValidatorResult> results = new ArrayList<>();
if (!canReadConfigurationFile(results)) {
return results;
}
try {
final Pattern pattern = getPattern();
final Matcher matcher = pattern.matcher(getContents());
performChecks(matcher, results);
} catch (final IOException e) {
final RuntimeValidatorResult result = getResultBuilder(RuntimeValidatorResult.Outcome.FAILED)
.explanation(String.format("Configuration file [%s] cannot be read", getConfigurationFile().getAbsolutePath()))
.build();
results.add(result);
}
processResults(results);
return results;
}
protected RuntimeValidatorResult.Builder getResultBuilder(final RuntimeValidatorResult.Outcome outcome) {
return new RuntimeValidatorResult.Builder().subject(getClass().getSimpleName()).outcome(outcome);
}
protected File getConfigurationFile() {
return configurationFile;
}
protected String getContents() throws IOException {
// using Scanner to read file because reading whole lines for virtual files
// in Linux /proc/sys directory fail when using other implementations
try (Scanner scanner = new Scanner(configurationFile)) {
final StringBuilder builder = new StringBuilder();
while (scanner.hasNextLine()) {
builder.append(scanner.nextLine());
builder.append(System.lineSeparator());
}
return builder.toString();
}
}
protected abstract Pattern getPattern();
protected abstract void performChecks(final Matcher matcher, final List<RuntimeValidatorResult> results);
private boolean canReadConfigurationFile(final List<RuntimeValidatorResult> results) {
final File configurationFile = getConfigurationFile();
if (configurationFile == null) {
final RuntimeValidatorResult result = getResultBuilder(RuntimeValidatorResult.Outcome.SKIPPED)
.explanation("Configuration file not found")
.build();
results.add(result);
return false;
}
if (!configurationFile.canRead()) {
final RuntimeValidatorResult result = getResultBuilder(RuntimeValidatorResult.Outcome.SKIPPED)
.explanation(String.format("Configuration file [%s] cannot be read", configurationFile.getAbsolutePath()))
.build();
results.add(result);
return false;
}
return true;
}
private void processResults(final List<RuntimeValidatorResult> results) {
if (results.isEmpty()) {
final RuntimeValidatorResult result = getResultBuilder(RuntimeValidatorResult.Outcome.SUCCESSFUL).build();
results.add(result);
}
}
}

62
nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/process/AvailableLocalPorts.java Normal file
View File

@ -0,0 +1,62 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.bootstrap.process;
import java.io.File;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class AvailableLocalPorts extends AbstractFileBasedRuntimeValidator {
private static final String FILE_PATH = "/proc/sys/net/ipv4/ip_local_port_range";
private static final Pattern PATTERN = Pattern.compile("(\\d+)\\s+(\\d+)");
private static final int RECOMMENDED_AVAILABLE_PORTS = 55000;
public AvailableLocalPorts() {
super(new File(FILE_PATH));
}
AvailableLocalPorts(final File configurationFile) {
super(configurationFile);
}
@Override
protected Pattern getPattern() {
return PATTERN;
}
@Override
protected void performChecks(final Matcher matcher, final List<RuntimeValidatorResult> results) {
final String configurationPath = getConfigurationFile().getPath();
if (matcher.find()) {
final int lowerPort = Integer.parseInt(matcher.group(1));
final int higherPort = Integer.parseInt(matcher.group(2));
final int availablePorts = higherPort - lowerPort;
if (availablePorts < RECOMMENDED_AVAILABLE_PORTS) {
final RuntimeValidatorResult result = getResultBuilder(RuntimeValidatorResult.Outcome.FAILED)
.explanation(String.format("Local Ports [%d] less than recommended [%d] according to [%s]", availablePorts, RECOMMENDED_AVAILABLE_PORTS, configurationPath))
.build();
results.add(result);
}
} else {
final RuntimeValidatorResult result = getResultBuilder(RuntimeValidatorResult.Outcome.FAILED)
.explanation(String.format("Configuration file [%s] cannot be parsed", configurationPath))
.build();
results.add(result);
}
}
}

70
nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/process/FileHandles.java Normal file
View File

@ -0,0 +1,70 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.bootstrap.process;
import java.io.File;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class FileHandles extends AbstractFileBasedRuntimeValidator {
private static final String FILE_PATH = String.format("/proc/%s/limits", ProcessHandle.current().pid());
private static final Pattern PATTERN = Pattern.compile("Max open files\\s+(\\d+)\\s+(\\d+)\\s+files\\s+");
private static final int RECOMMENDED_SOFT_LIMIT = 50000;
private static final int RECOMMENDED_HARD_LIMIT = 50000;
public FileHandles() {
super(new File(FILE_PATH));
}
FileHandles(final File configurationFile) {
super(configurationFile);
}
@Override
protected Pattern getPattern() {
return PATTERN;
}
@Override
protected void performChecks(final Matcher matcher, final List<RuntimeValidatorResult> results) {
final String configurationPath = getConfigurationFile().getPath();
if (matcher.find()) {
final int softLimit = Integer.parseInt(matcher.group(1));
final int hardLimit = Integer.parseInt(matcher.group(2));
if (softLimit < RECOMMENDED_SOFT_LIMIT) {
final RuntimeValidatorResult result = getResultBuilder(RuntimeValidatorResult.Outcome.FAILED)
.explanation(String.format("Soft limit [%d] less than recommended [%d] according to [%s]", softLimit, RECOMMENDED_SOFT_LIMIT, configurationPath))
.build();
results.add(result);
}
if (hardLimit < RECOMMENDED_HARD_LIMIT) {
final RuntimeValidatorResult result = new RuntimeValidatorResult.Builder()
.subject(getClass().getSimpleName())
.outcome(RuntimeValidatorResult.Outcome.FAILED)
.explanation(String.format("Hard limit [%d] less than recommended [%d] according to [%s]", hardLimit, RECOMMENDED_HARD_LIMIT, configurationPath))
.build();
results.add(result);
}
} else {
final RuntimeValidatorResult result = getResultBuilder(RuntimeValidatorResult.Outcome.FAILED)
.explanation(String.format("Configuration file [%s] cannot be parsed", configurationPath))
.build();
results.add(result);
}
}
}

68
nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/process/ForkedProcesses.java Normal file
View File

@ -0,0 +1,68 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.bootstrap.process;
import java.io.File;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class ForkedProcesses extends AbstractFileBasedRuntimeValidator {
private static final String FILE_PATH = String.format("/proc/%s/limits", ProcessHandle.current().pid());
private static final Pattern PATTERN = Pattern.compile("Max processes\\s+(\\d+)\\s+(\\d+)\\s+processes\\s+");
private static final int RECOMMENDED_SOFT_LIMIT = 10000;
private static final int RECOMMENDED_HARD_LIMIT = 10000;
public ForkedProcesses() {
super(new File(FILE_PATH));
}
ForkedProcesses(final File configurationFile) {
super(configurationFile);
}
@Override
protected Pattern getPattern() {
return PATTERN;
}
@Override
protected void performChecks(final Matcher matcher, final List<RuntimeValidatorResult> results) {
final String configurationPath = getConfigurationFile().getPath();
if (matcher.find()) {
final int softLimit = Integer.parseInt(matcher.group(1));
final int hardLimit = Integer.parseInt(matcher.group(2));
if (softLimit < RECOMMENDED_SOFT_LIMIT) {
final RuntimeValidatorResult result = getResultBuilder(RuntimeValidatorResult.Outcome.FAILED)
.explanation(String.format("Soft limit [%d] less than recommended [%d] according to [%s]", softLimit, RECOMMENDED_SOFT_LIMIT, configurationPath))
.build();
results.add(result);
}
if (hardLimit < RECOMMENDED_HARD_LIMIT) {
final RuntimeValidatorResult result = getResultBuilder(RuntimeValidatorResult.Outcome.FAILED)
.explanation(String.format("Hard limit [%d] less than recommended [%d] according to [%s]", hardLimit, RECOMMENDED_HARD_LIMIT, configurationPath))
.build();
results.add(result);
}
} else {
final RuntimeValidatorResult result = getResultBuilder(RuntimeValidatorResult.Outcome.FAILED)
.explanation(String.format("Configuration file [%s] cannot be parsed", configurationPath))
.build();
results.add(result);
}
}
}

28
nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/process/RuntimeValidator.java Normal file
View File

@ -0,0 +1,28 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.bootstrap.process;
import java.util.List;
public interface RuntimeValidator {
/**
* Validates if the given runtime configuration is within application best practices
*
* @return a List of {@code RuntimeValidatorResult}
*/
List<RuntimeValidatorResult> validate();
}

69
nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/process/RuntimeValidatorExecutor.java Normal file
View File

@ -0,0 +1,69 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.bootstrap.process;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.stream.Collectors;
public class RuntimeValidatorExecutor {
private final List<RuntimeValidator> configurationClasses;
private static final Logger logger = LoggerFactory.getLogger(RuntimeValidatorExecutor.class);
public RuntimeValidatorExecutor() {
this.configurationClasses = Arrays.asList(
new AvailableLocalPorts(),
new FileHandles(),
new ForkedProcesses(),
new Swappiness(),
new SocketTimedWaitDuration()
);
}
RuntimeValidatorExecutor(final List<RuntimeValidator> configurationClasses) {
this.configurationClasses = configurationClasses;
}
/**
* Checks all the system configuration settings that are supported to be checked
*/
public List<RuntimeValidatorResult> execute() {
final List<RuntimeValidatorResult> results = new ArrayList<>();
for (final RuntimeValidator configuration: configurationClasses) {
results.addAll(configuration.validate());
}
final List<RuntimeValidatorResult> failures = results
.stream()
.filter((result) -> result.getOutcome().equals(RuntimeValidatorResult.Outcome.FAILED))
.collect(Collectors.toList());
if (!failures.isEmpty()) {
logWarnings(failures);
}
return results;
}
private void logWarnings(final List<RuntimeValidatorResult> results) {
for (final RuntimeValidatorResult result : results) {
logger.warn("Runtime Configuration [{}] validation failed: {}", result.getSubject(), result.getExplanation());
}
}
}

77
nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/process/RuntimeValidatorResult.java Normal file
View File

@ -0,0 +1,77 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.bootstrap.process;
public class RuntimeValidatorResult {
private final String subject;
private final String explanation;
private final Outcome outcome;
protected RuntimeValidatorResult(final Builder builder) {
this.subject = builder.subject;
this.explanation = builder.explanation;
this.outcome = builder.outcome;
}
public String getSubject() {
return subject;
}
public String getExplanation() {
return explanation;
}
public Outcome getOutcome() {
return outcome;
}
public static final class Builder {
private String subject = "";
private String explanation = "";
private Outcome outcome = Outcome.FAILED;
public Builder subject(final String subject) {
if (subject != null) {
this.subject = subject;
}
return this;
}
public Builder explanation(final String explanation) {
if (explanation != null) {
this.explanation = explanation;
}
return this;
}
public Builder outcome(final Outcome outcome) {
this.outcome = outcome;
return this;
}
public RuntimeValidatorResult build() {
return new RuntimeValidatorResult(this);
}
}
public enum Outcome {
SUCCESSFUL,
FAILED,
SKIPPED;
}
}

82
nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/process/SocketTimedWaitDuration.java Normal file
View File

@ -0,0 +1,82 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.bootstrap.process;
import java.io.File;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class SocketTimedWaitDuration extends AbstractFileBasedRuntimeValidator {
// Different Linux Kernel versions have different file paths for TIMED_WAIT_DURATION
private static final String[] POSSIBLE_FILE_PATHS = new String[] {
"/proc/sys/net/ipv4/tcp_tw_timeout",
"/proc/sys/net/netfilter/nf_conntrack_tcp_timeout_time_wait",
"/proc/sys/net/ipv4/netfilter/ip_conntrack_tcp_timeout_time_wait"
};
private static final Pattern PATTERN = Pattern.compile("\\d+");
private static final int DESIRED_TIMED_WAIT_DURATION = 1;
public SocketTimedWaitDuration() {
super(determineConfigurationFile());
}
SocketTimedWaitDuration(final File configurationFile) {
super(configurationFile);
}
@Override
protected Pattern getPattern() {
return PATTERN;
}
@Override
protected void performChecks(final Matcher matcher, final List<RuntimeValidatorResult> results) {
final String configurationPath = getConfigurationFile().getAbsolutePath();
if (matcher.find()) {
final int timedWaitDuration = Integer.parseInt(matcher.group());
if (timedWaitDuration > DESIRED_TIMED_WAIT_DURATION) {
final RuntimeValidatorResult result = getResultBuilder(RuntimeValidatorResult.Outcome.FAILED)
.explanation(
String.format(
"TCP Socket Wait [%d seconds] more than recommended [%d seconds] according to [%s]",
timedWaitDuration,
DESIRED_TIMED_WAIT_DURATION,
configurationPath
)
)
.build();
results.add(result);
}
} else {
final RuntimeValidatorResult result = getResultBuilder(RuntimeValidatorResult.Outcome.FAILED)
.explanation(String.format("Configuration file [%s] cannot be parsed", getConfigurationFile().getAbsolutePath()))
.build();
results.add(result);
}
}
private static File determineConfigurationFile() {
for (final String filePath: POSSIBLE_FILE_PATHS) {
final File file = new File(filePath);
if (file.canRead()) {
return file;
}
}
return null;
}
}

60
nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/process/Swappiness.java Normal file
View File

@ -0,0 +1,60 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.bootstrap.process;
import java.io.File;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class Swappiness extends AbstractFileBasedRuntimeValidator {
private static final String FILE_PATH = "/proc/sys/vm/swappiness";
private static final Pattern PATTERN = Pattern.compile("\\d+");
private static final int RECOMMENDED_SWAPPINESS = 0;
public Swappiness() {
super(new File(FILE_PATH));
}
Swappiness(final File configurationFile) {
super(configurationFile);
}
@Override
protected Pattern getPattern() {
return PATTERN;
}
@Override
protected void performChecks(final Matcher matcher, final List<RuntimeValidatorResult> results) {
final String configurationPath = getConfigurationFile().getAbsolutePath();
if (matcher.find()) {
final int swappiness = Integer.parseInt(matcher.group());
if (swappiness > RECOMMENDED_SWAPPINESS) {
final RuntimeValidatorResult result = getResultBuilder(RuntimeValidatorResult.Outcome.FAILED)
.explanation(String.format("Swappiness [%d] more than recommended [%d] according to [%s]", swappiness, RECOMMENDED_SWAPPINESS, configurationPath))
.build();
results.add(result);
}
} else {
final RuntimeValidatorResult result = getResultBuilder(RuntimeValidatorResult.Outcome.FAILED)
.explanation(String.format("Configuration file [%s] cannot be parsed", configurationPath))
.build();
results.add(result);
}
}
}

201
nifi-bootstrap/src/test/java/org/apache/nifi/bootstrap/process/RuntimeValidatorExecutorTest.java Normal file
View File

@ -0,0 +1,201 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.bootstrap.process;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import java.io.File;
import java.io.IOException;
import java.net.URL;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
public class RuntimeValidatorExecutorTest {
@TempDir
private File tempDir;
private RuntimeValidatorExecutor runtimeValidatorExecutor;
@Test
public void testAllSatisfactory() throws IOException {
final List<RuntimeValidator> configurationClasses = getAllTestConfigurationClasses();
runtimeValidatorExecutor = new RuntimeValidatorExecutor(configurationClasses);
final List<RuntimeValidatorResult> results = runtimeValidatorExecutor.execute();
assertEquals(5, results.size());
final List<RuntimeValidatorResult> failures = getFailures(results);
assertEquals(0, failures.size());
}
@Test
public void testAllFailuresEmptyFiles() throws IOException {
final File emptyFile = getTempFile("empty_file", "");
runtimeValidatorExecutor = new RuntimeValidatorExecutor(getConfigurationClassesWithFile(emptyFile));
final List<RuntimeValidatorResult> results = runtimeValidatorExecutor.execute();
assertEquals(5, results.size());
final List<RuntimeValidatorResult> failures = getFailures(results);
assertEquals(5, failures.size());
for (final RuntimeValidatorResult failure : failures) {
assertTrue(failure.getExplanation().contains("parse"));
}
}
@Test
public void testAllFailuresUnparsable() throws IOException {
final File unparsableFile = getTempFile("unparsable", "abcdefghijklmnopqrstuvwxyz");
runtimeValidatorExecutor = new RuntimeValidatorExecutor(getConfigurationClassesWithFile(unparsableFile));
final List<RuntimeValidatorResult> results = runtimeValidatorExecutor.execute();
assertEquals(5, results.size());
final List<RuntimeValidatorResult> failures = getFailures(results);
assertEquals(5, failures.size());
for (final RuntimeValidatorResult failure : failures) {
assertTrue(failure.getExplanation().contains("parse"));
}
}
@Test
public void testCannotFindFilesForConfiguration() {
final File missingFile = new File("missing_file");
runtimeValidatorExecutor = new RuntimeValidatorExecutor(getConfigurationClassesWithFile(missingFile));
final List<RuntimeValidatorResult> results = runtimeValidatorExecutor.execute();
assertEquals(5, results.size());
final List<RuntimeValidatorResult> skipped = getSkipped(results);
assertEquals(5, skipped.size());
for (final RuntimeValidatorResult result : skipped) {
assertTrue(result.getExplanation().contains("read"));
}
}
@Test
public void testNotEnoughAvailablePorts() throws IOException {
final List<RuntimeValidator> configurationClasses = new ArrayList<>();
configurationClasses.add(new AvailableLocalPorts(getTempFile("available_ports_not_enough", "0 1")));
runtimeValidatorExecutor = new RuntimeValidatorExecutor(configurationClasses);
final List<RuntimeValidatorResult> results = runtimeValidatorExecutor.execute();
assertEquals(1, results.size());
final List<RuntimeValidatorResult> failures = getFailures(results);
assertEquals(1, failures.size());
for (final RuntimeValidatorResult failure : failures) {
assertTrue(failure.getExplanation().contains("less than"));
}
}
@Test
public void testNotEnoughFileHandlesAndForkedProcesses() {
final List<RuntimeValidator> configurationClasses = new ArrayList<>();
configurationClasses.add(new FileHandles(getTestFile("limits_not_enough")));
configurationClasses.add(new ForkedProcesses(getTestFile("limits_not_enough")));
runtimeValidatorExecutor = new RuntimeValidatorExecutor(configurationClasses);
final List<RuntimeValidatorResult> results = runtimeValidatorExecutor.execute();
assertEquals(4, results.size());
final List<RuntimeValidatorResult> failures = getFailures(results);
assertEquals(4, failures.size());
for (final RuntimeValidatorResult failure : failures) {
assertTrue(failure.getExplanation().contains("less than"));
}
}
@Test
public void testHighSwappiness() throws IOException {
final List<RuntimeValidator> configurationClasses = new ArrayList<>();
configurationClasses.add(new Swappiness(getTempFile("swappiness_high", "50")));
runtimeValidatorExecutor = new RuntimeValidatorExecutor(configurationClasses);
final List<RuntimeValidatorResult> results = runtimeValidatorExecutor.execute();
assertEquals(1, results.size());
final List<RuntimeValidatorResult> failures = getFailures(results);
assertEquals(1, failures.size());
for (final RuntimeValidatorResult failure : failures) {
assertTrue(failure.getExplanation().contains("more than"));
}
}
@Test
public void testHighTimedWaitDuration() throws IOException {
final List<RuntimeValidator> configurationClasses = new ArrayList<>();
configurationClasses.add(new SocketTimedWaitDuration(getTempFile("tcp_tw_timeout_high", "50")));
runtimeValidatorExecutor = new RuntimeValidatorExecutor(configurationClasses);
final List<RuntimeValidatorResult> results = runtimeValidatorExecutor.execute();
assertEquals(1, results.size());
final List<RuntimeValidatorResult> failures = getFailures(results);
assertEquals(1, failures.size());
for (final RuntimeValidatorResult failure : failures) {
assertTrue(failure.getExplanation().contains("more than"));
}
}
private List<RuntimeValidatorResult> getFailures(final List<RuntimeValidatorResult> results) {
return results
.stream()
.filter((result) -> result.getOutcome().equals(RuntimeValidatorResult.Outcome.FAILED))
.collect(Collectors.toList());
}
private List<RuntimeValidatorResult> getSkipped(final List<RuntimeValidatorResult> results) {
return results
.stream()
.filter((result) -> result.getOutcome().equals(RuntimeValidatorResult.Outcome.SKIPPED))
.collect(Collectors.toList());
}
private File getTestFile(final String filename) {
final ClassLoader classLoader = this.getClass().getClassLoader();
final URL url = classLoader.getResource(filename);
if (url == null) {
throw new IllegalStateException(String.format("File [%s] not found", filename));
}
return new File(url.getFile());
}
private File getTempFile(final String fileName, final String text) throws IOException {
final File tempFile = new File(tempDir, fileName);
Files.write(tempFile.toPath(), text.getBytes());
return tempFile;
}
private List<RuntimeValidator> getAllTestConfigurationClasses() throws IOException {
final List<RuntimeValidator> configurationClasses = new ArrayList<>();
configurationClasses.add(new AvailableLocalPorts(getTempFile("available_ports", "1 550001")));
configurationClasses.add(new FileHandles(getTestFile("limits")));
configurationClasses.add(new ForkedProcesses(getTestFile("limits")));
configurationClasses.add(new Swappiness(getTempFile("swappiness", "0")));
configurationClasses.add(new SocketTimedWaitDuration(getTempFile("tcp_tw_timeout", "1")));
return configurationClasses;
}
private List<RuntimeValidator> getConfigurationClassesWithFile(final File file) {
final List<RuntimeValidator> configurationClasses = new ArrayList<>();
configurationClasses.add(new AvailableLocalPorts(file));
configurationClasses.add(new FileHandles(file));
configurationClasses.add(new ForkedProcesses(file));
configurationClasses.add(new Swappiness(file));
configurationClasses.add(new SocketTimedWaitDuration(file));
return configurationClasses;
}
}

31
nifi-bootstrap/src/test/resources/limits Normal file
View File

@ -0,0 +1,31 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
Limit Soft Limit Hard Limit Units
Max cpu time unlimited unlimited seconds
Max file size unlimited unlimited bytes
Max data size unlimited unlimited bytes
Max stack size 8388608 unlimited bytes
Max core file size 0 unlimited bytes
Max resident set unlimited unlimited bytes
Max processes 62978 62978 processes
Max open files 1048576 1048576 files
Max locked memory 2078633984 2078633984 bytes
Max address space unlimited unlimited bytes
Max file locks unlimited unlimited locks
Max pending signals 62978 62978 signals
Max msgqueue size 819200 819200 bytes
Max nice priority 0 0
Max realtime priority 0 0
Max realtime timeout unlimited unlimited us

31
nifi-bootstrap/src/test/resources/limits_not_enough Normal file
View File

@ -0,0 +1,31 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
Limit Soft Limit Hard Limit Units
Max cpu time unlimited unlimited seconds
Max file size unlimited unlimited bytes
Max data size unlimited unlimited bytes
Max stack size 8388608 unlimited bytes
Max core file size 0 unlimited bytes
Max resident set unlimited unlimited bytes
Max processes 5000 5000 processes
Max open files 1000 1000 files
Max locked memory 2078633984 2078633984 bytes
Max address space unlimited unlimited bytes
Max file locks unlimited unlimited locks
Max pending signals 62978 62978 signals
Max msgqueue size 819200 819200 bytes
Max nice priority 0 0
Max realtime priority 0 0
Max realtime timeout unlimited unlimited us