diff --git a/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/RunNiFi.java b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/RunNiFi.java index abea96dcff..637d57e22f 100644 --- a/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/RunNiFi.java +++ b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/RunNiFi.java @@ -18,6 +18,7 @@ package org.apache.nifi.bootstrap; import org.apache.commons.lang3.StringUtils; import org.apache.nifi.bootstrap.notification.NotificationType; +import org.apache.nifi.bootstrap.process.RuntimeValidatorExecutor; import org.apache.nifi.bootstrap.util.DumpFileValidator; import org.apache.nifi.bootstrap.util.SecureNiFiConfigUtil; import org.apache.nifi.util.file.FileUtils; @@ -146,6 +147,7 @@ public class RunNiFi { private final Logger defaultLogger = LoggerFactory.getLogger(RunNiFi.class); private final ExecutorService loggingExecutor; + private final RuntimeValidatorExecutor runtimeValidatorExecutor; private volatile Set> loggingFutures = new HashSet<>(2); private final NotificationServiceManager serviceManager; @@ -163,6 +165,8 @@ public class RunNiFi { }); serviceManager = loadServices(); + + runtimeValidatorExecutor = new RuntimeValidatorExecutor(); } private static void printUsage() { @@ -1113,6 +1117,8 @@ public class RunNiFi { cmdLogger.warn("Failed to delete previous lock file {}; this file should be cleaned up manually", prevLockFile); } + runtimeValidatorExecutor.execute(); + final ProcessBuilder builder = new ProcessBuilder(); if (!bootstrapConfigFile.exists()) { diff --git a/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/process/AbstractFileBasedRuntimeValidator.java b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/process/AbstractFileBasedRuntimeValidator.java new file mode 100644 index 0000000000..de2c123dcf --- /dev/null +++ b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/process/AbstractFileBasedRuntimeValidator.java @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.bootstrap.process; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Scanner; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public abstract class AbstractFileBasedRuntimeValidator implements RuntimeValidator { + private final File configurationFile; + + AbstractFileBasedRuntimeValidator(final File configurationFile) { + this.configurationFile = configurationFile; + } + + @Override + public List validate() { + final List results = new ArrayList<>(); + if (!canReadConfigurationFile(results)) { + return results; + } + + try { + final Pattern pattern = getPattern(); + final Matcher matcher = pattern.matcher(getContents()); + performChecks(matcher, results); + } catch (final IOException e) { + final RuntimeValidatorResult result = getResultBuilder(RuntimeValidatorResult.Outcome.FAILED) + .explanation(String.format("Configuration file [%s] cannot be read", getConfigurationFile().getAbsolutePath())) + .build(); + results.add(result); + } + + processResults(results); + return results; + } + + protected RuntimeValidatorResult.Builder getResultBuilder(final RuntimeValidatorResult.Outcome outcome) { + return new RuntimeValidatorResult.Builder().subject(getClass().getSimpleName()).outcome(outcome); + } + + protected File getConfigurationFile() { + return configurationFile; + } + + protected String getContents() throws IOException { + // using Scanner to read file because reading whole lines for virtual files + // in Linux /proc/sys directory fail when using other implementations + try (Scanner scanner = new Scanner(configurationFile)) { + final StringBuilder builder = new StringBuilder(); + while (scanner.hasNextLine()) { + builder.append(scanner.nextLine()); + builder.append(System.lineSeparator()); + } + return builder.toString(); + } + } + + protected abstract Pattern getPattern(); + + protected abstract void performChecks(final Matcher matcher, final List results); + + private boolean canReadConfigurationFile(final List results) { + final File configurationFile = getConfigurationFile(); + if (configurationFile == null) { + final RuntimeValidatorResult result = getResultBuilder(RuntimeValidatorResult.Outcome.SKIPPED) + .explanation("Configuration file not found") + .build(); + results.add(result); + return false; + } + if (!configurationFile.canRead()) { + final RuntimeValidatorResult result = getResultBuilder(RuntimeValidatorResult.Outcome.SKIPPED) + .explanation(String.format("Configuration file [%s] cannot be read", configurationFile.getAbsolutePath())) + .build(); + results.add(result); + return false; + } + return true; + } + + private void processResults(final List results) { + if (results.isEmpty()) { + final RuntimeValidatorResult result = getResultBuilder(RuntimeValidatorResult.Outcome.SUCCESSFUL).build(); + results.add(result); + } + } +} diff --git a/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/process/AvailableLocalPorts.java b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/process/AvailableLocalPorts.java new file mode 100644 index 0000000000..33d4c7a328 --- /dev/null +++ b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/process/AvailableLocalPorts.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.bootstrap.process; + +import java.io.File; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class AvailableLocalPorts extends AbstractFileBasedRuntimeValidator { + private static final String FILE_PATH = "/proc/sys/net/ipv4/ip_local_port_range"; + private static final Pattern PATTERN = Pattern.compile("(\\d+)\\s+(\\d+)"); + private static final int RECOMMENDED_AVAILABLE_PORTS = 55000; + + public AvailableLocalPorts() { + super(new File(FILE_PATH)); + } + + AvailableLocalPorts(final File configurationFile) { + super(configurationFile); + } + + @Override + protected Pattern getPattern() { + return PATTERN; + } + + @Override + protected void performChecks(final Matcher matcher, final List results) { + final String configurationPath = getConfigurationFile().getPath(); + if (matcher.find()) { + final int lowerPort = Integer.parseInt(matcher.group(1)); + final int higherPort = Integer.parseInt(matcher.group(2)); + final int availablePorts = higherPort - lowerPort; + if (availablePorts < RECOMMENDED_AVAILABLE_PORTS) { + final RuntimeValidatorResult result = getResultBuilder(RuntimeValidatorResult.Outcome.FAILED) + .explanation(String.format("Local Ports [%d] less than recommended [%d] according to [%s]", availablePorts, RECOMMENDED_AVAILABLE_PORTS, configurationPath)) + .build(); + results.add(result); + } + } else { + final RuntimeValidatorResult result = getResultBuilder(RuntimeValidatorResult.Outcome.FAILED) + .explanation(String.format("Configuration file [%s] cannot be parsed", configurationPath)) + .build(); + results.add(result); + } + } +} diff --git a/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/process/FileHandles.java b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/process/FileHandles.java new file mode 100644 index 0000000000..dfb80a1f64 --- /dev/null +++ b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/process/FileHandles.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.bootstrap.process; + +import java.io.File; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class FileHandles extends AbstractFileBasedRuntimeValidator { + private static final String FILE_PATH = String.format("/proc/%s/limits", ProcessHandle.current().pid()); + private static final Pattern PATTERN = Pattern.compile("Max open files\\s+(\\d+)\\s+(\\d+)\\s+files\\s+"); + private static final int RECOMMENDED_SOFT_LIMIT = 50000; + private static final int RECOMMENDED_HARD_LIMIT = 50000; + + public FileHandles() { + super(new File(FILE_PATH)); + } + + FileHandles(final File configurationFile) { + super(configurationFile); + } + + @Override + protected Pattern getPattern() { + return PATTERN; + } + + @Override + protected void performChecks(final Matcher matcher, final List results) { + final String configurationPath = getConfigurationFile().getPath(); + if (matcher.find()) { + final int softLimit = Integer.parseInt(matcher.group(1)); + final int hardLimit = Integer.parseInt(matcher.group(2)); + if (softLimit < RECOMMENDED_SOFT_LIMIT) { + final RuntimeValidatorResult result = getResultBuilder(RuntimeValidatorResult.Outcome.FAILED) + .explanation(String.format("Soft limit [%d] less than recommended [%d] according to [%s]", softLimit, RECOMMENDED_SOFT_LIMIT, configurationPath)) + .build(); + results.add(result); + } + if (hardLimit < RECOMMENDED_HARD_LIMIT) { + final RuntimeValidatorResult result = new RuntimeValidatorResult.Builder() + .subject(getClass().getSimpleName()) + .outcome(RuntimeValidatorResult.Outcome.FAILED) + .explanation(String.format("Hard limit [%d] less than recommended [%d] according to [%s]", hardLimit, RECOMMENDED_HARD_LIMIT, configurationPath)) + .build(); + results.add(result); + } + } else { + final RuntimeValidatorResult result = getResultBuilder(RuntimeValidatorResult.Outcome.FAILED) + .explanation(String.format("Configuration file [%s] cannot be parsed", configurationPath)) + .build(); + results.add(result); + } + } +} diff --git a/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/process/ForkedProcesses.java b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/process/ForkedProcesses.java new file mode 100644 index 0000000000..52a0ed56c5 --- /dev/null +++ b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/process/ForkedProcesses.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.bootstrap.process; + +import java.io.File; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class ForkedProcesses extends AbstractFileBasedRuntimeValidator { + private static final String FILE_PATH = String.format("/proc/%s/limits", ProcessHandle.current().pid()); + private static final Pattern PATTERN = Pattern.compile("Max processes\\s+(\\d+)\\s+(\\d+)\\s+processes\\s+"); + private static final int RECOMMENDED_SOFT_LIMIT = 10000; + private static final int RECOMMENDED_HARD_LIMIT = 10000; + + public ForkedProcesses() { + super(new File(FILE_PATH)); + } + + ForkedProcesses(final File configurationFile) { + super(configurationFile); + } + + @Override + protected Pattern getPattern() { + return PATTERN; + } + + @Override + protected void performChecks(final Matcher matcher, final List results) { + final String configurationPath = getConfigurationFile().getPath(); + if (matcher.find()) { + final int softLimit = Integer.parseInt(matcher.group(1)); + final int hardLimit = Integer.parseInt(matcher.group(2)); + if (softLimit < RECOMMENDED_SOFT_LIMIT) { + final RuntimeValidatorResult result = getResultBuilder(RuntimeValidatorResult.Outcome.FAILED) + .explanation(String.format("Soft limit [%d] less than recommended [%d] according to [%s]", softLimit, RECOMMENDED_SOFT_LIMIT, configurationPath)) + .build(); + results.add(result); + } + if (hardLimit < RECOMMENDED_HARD_LIMIT) { + final RuntimeValidatorResult result = getResultBuilder(RuntimeValidatorResult.Outcome.FAILED) + .explanation(String.format("Hard limit [%d] less than recommended [%d] according to [%s]", hardLimit, RECOMMENDED_HARD_LIMIT, configurationPath)) + .build(); + results.add(result); + } + } else { + final RuntimeValidatorResult result = getResultBuilder(RuntimeValidatorResult.Outcome.FAILED) + .explanation(String.format("Configuration file [%s] cannot be parsed", configurationPath)) + .build(); + results.add(result); + } + } +} diff --git a/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/process/RuntimeValidator.java b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/process/RuntimeValidator.java new file mode 100644 index 0000000000..d4c0df2c6b --- /dev/null +++ b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/process/RuntimeValidator.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.bootstrap.process; + +import java.util.List; + +public interface RuntimeValidator { + /** + * Validates if the given runtime configuration is within application best practices + * + * @return a List of {@code RuntimeValidatorResult} + */ + List validate(); +} diff --git a/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/process/RuntimeValidatorExecutor.java b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/process/RuntimeValidatorExecutor.java new file mode 100644 index 0000000000..8ad0a88eb1 --- /dev/null +++ b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/process/RuntimeValidatorExecutor.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.bootstrap.process; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +public class RuntimeValidatorExecutor { + private final List configurationClasses; + private static final Logger logger = LoggerFactory.getLogger(RuntimeValidatorExecutor.class); + + public RuntimeValidatorExecutor() { + this.configurationClasses = Arrays.asList( + new AvailableLocalPorts(), + new FileHandles(), + new ForkedProcesses(), + new Swappiness(), + new SocketTimedWaitDuration() + ); + } + + RuntimeValidatorExecutor(final List configurationClasses) { + this.configurationClasses = configurationClasses; + } + + /** + * Checks all the system configuration settings that are supported to be checked + */ + public List execute() { + final List results = new ArrayList<>(); + for (final RuntimeValidator configuration: configurationClasses) { + results.addAll(configuration.validate()); + } + final List failures = results + .stream() + .filter((result) -> result.getOutcome().equals(RuntimeValidatorResult.Outcome.FAILED)) + .collect(Collectors.toList()); + if (!failures.isEmpty()) { + logWarnings(failures); + } + + return results; + } + + private void logWarnings(final List results) { + for (final RuntimeValidatorResult result : results) { + logger.warn("Runtime Configuration [{}] validation failed: {}", result.getSubject(), result.getExplanation()); + } + } +} diff --git a/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/process/RuntimeValidatorResult.java b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/process/RuntimeValidatorResult.java new file mode 100644 index 0000000000..972a01dac7 --- /dev/null +++ b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/process/RuntimeValidatorResult.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.bootstrap.process; + +public class RuntimeValidatorResult { + private final String subject; + private final String explanation; + private final Outcome outcome; + protected RuntimeValidatorResult(final Builder builder) { + this.subject = builder.subject; + this.explanation = builder.explanation; + this.outcome = builder.outcome; + } + + public String getSubject() { + return subject; + } + + public String getExplanation() { + return explanation; + } + + public Outcome getOutcome() { + return outcome; + } + + public static final class Builder { + private String subject = ""; + private String explanation = ""; + private Outcome outcome = Outcome.FAILED; + + public Builder subject(final String subject) { + if (subject != null) { + this.subject = subject; + } + return this; + } + + public Builder explanation(final String explanation) { + if (explanation != null) { + this.explanation = explanation; + } + return this; + } + + public Builder outcome(final Outcome outcome) { + this.outcome = outcome; + return this; + } + + public RuntimeValidatorResult build() { + return new RuntimeValidatorResult(this); + } + } + + public enum Outcome { + SUCCESSFUL, + + FAILED, + + SKIPPED; + } +} diff --git a/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/process/SocketTimedWaitDuration.java b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/process/SocketTimedWaitDuration.java new file mode 100644 index 0000000000..0b9d0ddbad --- /dev/null +++ b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/process/SocketTimedWaitDuration.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.bootstrap.process; + +import java.io.File; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class SocketTimedWaitDuration extends AbstractFileBasedRuntimeValidator { + // Different Linux Kernel versions have different file paths for TIMED_WAIT_DURATION + private static final String[] POSSIBLE_FILE_PATHS = new String[] { + "/proc/sys/net/ipv4/tcp_tw_timeout", + "/proc/sys/net/netfilter/nf_conntrack_tcp_timeout_time_wait", + "/proc/sys/net/ipv4/netfilter/ip_conntrack_tcp_timeout_time_wait" + }; + private static final Pattern PATTERN = Pattern.compile("\\d+"); + private static final int DESIRED_TIMED_WAIT_DURATION = 1; + + public SocketTimedWaitDuration() { + super(determineConfigurationFile()); + } + + SocketTimedWaitDuration(final File configurationFile) { + super(configurationFile); + } + + @Override + protected Pattern getPattern() { + return PATTERN; + } + + @Override + protected void performChecks(final Matcher matcher, final List results) { + final String configurationPath = getConfigurationFile().getAbsolutePath(); + if (matcher.find()) { + final int timedWaitDuration = Integer.parseInt(matcher.group()); + if (timedWaitDuration > DESIRED_TIMED_WAIT_DURATION) { + final RuntimeValidatorResult result = getResultBuilder(RuntimeValidatorResult.Outcome.FAILED) + .explanation( + String.format( + "TCP Socket Wait [%d seconds] more than recommended [%d seconds] according to [%s]", + timedWaitDuration, + DESIRED_TIMED_WAIT_DURATION, + configurationPath + ) + ) + .build(); + results.add(result); + } + } else { + final RuntimeValidatorResult result = getResultBuilder(RuntimeValidatorResult.Outcome.FAILED) + .explanation(String.format("Configuration file [%s] cannot be parsed", getConfigurationFile().getAbsolutePath())) + .build(); + results.add(result); + } + } + + private static File determineConfigurationFile() { + for (final String filePath: POSSIBLE_FILE_PATHS) { + final File file = new File(filePath); + if (file.canRead()) { + return file; + } + } + return null; + } +} diff --git a/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/process/Swappiness.java b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/process/Swappiness.java new file mode 100644 index 0000000000..945bda4683 --- /dev/null +++ b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/process/Swappiness.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.bootstrap.process; + +import java.io.File; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class Swappiness extends AbstractFileBasedRuntimeValidator { + private static final String FILE_PATH = "/proc/sys/vm/swappiness"; + private static final Pattern PATTERN = Pattern.compile("\\d+"); + private static final int RECOMMENDED_SWAPPINESS = 0; + + public Swappiness() { + super(new File(FILE_PATH)); + } + + Swappiness(final File configurationFile) { + super(configurationFile); + } + + @Override + protected Pattern getPattern() { + return PATTERN; + } + + @Override + protected void performChecks(final Matcher matcher, final List results) { + final String configurationPath = getConfigurationFile().getAbsolutePath(); + if (matcher.find()) { + final int swappiness = Integer.parseInt(matcher.group()); + if (swappiness > RECOMMENDED_SWAPPINESS) { + final RuntimeValidatorResult result = getResultBuilder(RuntimeValidatorResult.Outcome.FAILED) + .explanation(String.format("Swappiness [%d] more than recommended [%d] according to [%s]", swappiness, RECOMMENDED_SWAPPINESS, configurationPath)) + .build(); + results.add(result); + } + } else { + final RuntimeValidatorResult result = getResultBuilder(RuntimeValidatorResult.Outcome.FAILED) + .explanation(String.format("Configuration file [%s] cannot be parsed", configurationPath)) + .build(); + results.add(result); + } + } +} diff --git a/nifi-bootstrap/src/test/java/org/apache/nifi/bootstrap/process/RuntimeValidatorExecutorTest.java b/nifi-bootstrap/src/test/java/org/apache/nifi/bootstrap/process/RuntimeValidatorExecutorTest.java new file mode 100644 index 0000000000..4aa57ab5dd --- /dev/null +++ b/nifi-bootstrap/src/test/java/org/apache/nifi/bootstrap/process/RuntimeValidatorExecutorTest.java @@ -0,0 +1,201 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.bootstrap.process; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.File; +import java.io.IOException; +import java.net.URL; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class RuntimeValidatorExecutorTest { + @TempDir + private File tempDir; + + private RuntimeValidatorExecutor runtimeValidatorExecutor; + + @Test + public void testAllSatisfactory() throws IOException { + final List configurationClasses = getAllTestConfigurationClasses(); + runtimeValidatorExecutor = new RuntimeValidatorExecutor(configurationClasses); + + final List results = runtimeValidatorExecutor.execute(); + assertEquals(5, results.size()); + final List failures = getFailures(results); + assertEquals(0, failures.size()); + } + + @Test + public void testAllFailuresEmptyFiles() throws IOException { + final File emptyFile = getTempFile("empty_file", ""); + runtimeValidatorExecutor = new RuntimeValidatorExecutor(getConfigurationClassesWithFile(emptyFile)); + + final List results = runtimeValidatorExecutor.execute(); + assertEquals(5, results.size()); + final List failures = getFailures(results); + assertEquals(5, failures.size()); + for (final RuntimeValidatorResult failure : failures) { + assertTrue(failure.getExplanation().contains("parse")); + } + } + + @Test + public void testAllFailuresUnparsable() throws IOException { + final File unparsableFile = getTempFile("unparsable", "abcdefghijklmnopqrstuvwxyz"); + runtimeValidatorExecutor = new RuntimeValidatorExecutor(getConfigurationClassesWithFile(unparsableFile)); + + final List results = runtimeValidatorExecutor.execute(); + assertEquals(5, results.size()); + final List failures = getFailures(results); + assertEquals(5, failures.size()); + for (final RuntimeValidatorResult failure : failures) { + assertTrue(failure.getExplanation().contains("parse")); + } + } + + @Test + public void testCannotFindFilesForConfiguration() { + final File missingFile = new File("missing_file"); + runtimeValidatorExecutor = new RuntimeValidatorExecutor(getConfigurationClassesWithFile(missingFile)); + + final List results = runtimeValidatorExecutor.execute(); + assertEquals(5, results.size()); + final List skipped = getSkipped(results); + assertEquals(5, skipped.size()); + for (final RuntimeValidatorResult result : skipped) { + assertTrue(result.getExplanation().contains("read")); + } + } + + @Test + public void testNotEnoughAvailablePorts() throws IOException { + final List configurationClasses = new ArrayList<>(); + configurationClasses.add(new AvailableLocalPorts(getTempFile("available_ports_not_enough", "0 1"))); + runtimeValidatorExecutor = new RuntimeValidatorExecutor(configurationClasses); + + final List results = runtimeValidatorExecutor.execute(); + assertEquals(1, results.size()); + final List failures = getFailures(results); + assertEquals(1, failures.size()); + for (final RuntimeValidatorResult failure : failures) { + assertTrue(failure.getExplanation().contains("less than")); + } + } + + @Test + public void testNotEnoughFileHandlesAndForkedProcesses() { + final List configurationClasses = new ArrayList<>(); + configurationClasses.add(new FileHandles(getTestFile("limits_not_enough"))); + configurationClasses.add(new ForkedProcesses(getTestFile("limits_not_enough"))); + runtimeValidatorExecutor = new RuntimeValidatorExecutor(configurationClasses); + + final List results = runtimeValidatorExecutor.execute(); + assertEquals(4, results.size()); + final List failures = getFailures(results); + assertEquals(4, failures.size()); + for (final RuntimeValidatorResult failure : failures) { + assertTrue(failure.getExplanation().contains("less than")); + } + } + + @Test + public void testHighSwappiness() throws IOException { + final List configurationClasses = new ArrayList<>(); + configurationClasses.add(new Swappiness(getTempFile("swappiness_high", "50"))); + runtimeValidatorExecutor = new RuntimeValidatorExecutor(configurationClasses); + + final List results = runtimeValidatorExecutor.execute(); + assertEquals(1, results.size()); + final List failures = getFailures(results); + assertEquals(1, failures.size()); + for (final RuntimeValidatorResult failure : failures) { + assertTrue(failure.getExplanation().contains("more than")); + } + } + + @Test + public void testHighTimedWaitDuration() throws IOException { + final List configurationClasses = new ArrayList<>(); + configurationClasses.add(new SocketTimedWaitDuration(getTempFile("tcp_tw_timeout_high", "50"))); + runtimeValidatorExecutor = new RuntimeValidatorExecutor(configurationClasses); + + final List results = runtimeValidatorExecutor.execute(); + assertEquals(1, results.size()); + final List failures = getFailures(results); + assertEquals(1, failures.size()); + for (final RuntimeValidatorResult failure : failures) { + assertTrue(failure.getExplanation().contains("more than")); + } + } + + private List getFailures(final List results) { + return results + .stream() + .filter((result) -> result.getOutcome().equals(RuntimeValidatorResult.Outcome.FAILED)) + .collect(Collectors.toList()); + } + + private List getSkipped(final List results) { + return results + .stream() + .filter((result) -> result.getOutcome().equals(RuntimeValidatorResult.Outcome.SKIPPED)) + .collect(Collectors.toList()); + } + + private File getTestFile(final String filename) { + final ClassLoader classLoader = this.getClass().getClassLoader(); + final URL url = classLoader.getResource(filename); + if (url == null) { + throw new IllegalStateException(String.format("File [%s] not found", filename)); + } + return new File(url.getFile()); + } + + private File getTempFile(final String fileName, final String text) throws IOException { + final File tempFile = new File(tempDir, fileName); + Files.write(tempFile.toPath(), text.getBytes()); + return tempFile; + } + + private List getAllTestConfigurationClasses() throws IOException { + final List configurationClasses = new ArrayList<>(); + configurationClasses.add(new AvailableLocalPorts(getTempFile("available_ports", "1 550001"))); + configurationClasses.add(new FileHandles(getTestFile("limits"))); + configurationClasses.add(new ForkedProcesses(getTestFile("limits"))); + configurationClasses.add(new Swappiness(getTempFile("swappiness", "0"))); + configurationClasses.add(new SocketTimedWaitDuration(getTempFile("tcp_tw_timeout", "1"))); + return configurationClasses; + } + + private List getConfigurationClassesWithFile(final File file) { + final List configurationClasses = new ArrayList<>(); + configurationClasses.add(new AvailableLocalPorts(file)); + configurationClasses.add(new FileHandles(file)); + configurationClasses.add(new ForkedProcesses(file)); + configurationClasses.add(new Swappiness(file)); + configurationClasses.add(new SocketTimedWaitDuration(file)); + return configurationClasses; + } +} diff --git a/nifi-bootstrap/src/test/resources/limits b/nifi-bootstrap/src/test/resources/limits new file mode 100644 index 0000000000..c93a765447 --- /dev/null +++ b/nifi-bootstrap/src/test/resources/limits @@ -0,0 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +Limit Soft Limit Hard Limit Units +Max cpu time unlimited unlimited seconds +Max file size unlimited unlimited bytes +Max data size unlimited unlimited bytes +Max stack size 8388608 unlimited bytes +Max core file size 0 unlimited bytes +Max resident set unlimited unlimited bytes +Max processes 62978 62978 processes +Max open files 1048576 1048576 files +Max locked memory 2078633984 2078633984 bytes +Max address space unlimited unlimited bytes +Max file locks unlimited unlimited locks +Max pending signals 62978 62978 signals +Max msgqueue size 819200 819200 bytes +Max nice priority 0 0 +Max realtime priority 0 0 +Max realtime timeout unlimited unlimited us \ No newline at end of file diff --git a/nifi-bootstrap/src/test/resources/limits_not_enough b/nifi-bootstrap/src/test/resources/limits_not_enough new file mode 100644 index 0000000000..25811db718 --- /dev/null +++ b/nifi-bootstrap/src/test/resources/limits_not_enough @@ -0,0 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +Limit Soft Limit Hard Limit Units +Max cpu time unlimited unlimited seconds +Max file size unlimited unlimited bytes +Max data size unlimited unlimited bytes +Max stack size 8388608 unlimited bytes +Max core file size 0 unlimited bytes +Max resident set unlimited unlimited bytes +Max processes 5000 5000 processes +Max open files 1000 1000 files +Max locked memory 2078633984 2078633984 bytes +Max address space unlimited unlimited bytes +Max file locks unlimited unlimited locks +Max pending signals 62978 62978 signals +Max msgqueue size 819200 819200 bytes +Max nice priority 0 0 +Max realtime priority 0 0 +Max realtime timeout unlimited unlimited us \ No newline at end of file