diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/OzoneClientUtils.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/OzoneClientUtils.java index 80a2d33ca2a..549dc804c54 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/OzoneClientUtils.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/OzoneClientUtils.java @@ -32,6 +32,12 @@ import java.util.HashMap; import java.util.Map; import static org.apache.hadoop.ozone.OzoneConfigKeys.*; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SCM_DEADNODE_INTERVAL_DEFAULT; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SCM_DEADNODE_INTERVAL_MS; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SCM_HEARTBEAT_INTERVAL_SECONDS; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL_MS; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SCM_STALENODE_INTERVAL_DEFAULT; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SCM_STALENODE_INTERVAL_MS; /** * Utility methods for Ozone and Container Clients. @@ -238,4 +244,138 @@ public final class OzoneClientUtils { services.put(OZONE_SCM_SERVICE_ID, serviceInstances); return services; } + + /** + * Checks that a given value is with a range. + * + * For example, sanitizeUserArgs(17, 3, 5, 10) + * ensures that 17 is greater/equal than 3 * 5 and less/equal to 3 * 10. + * + * @param valueTocheck - value to check + * @param baseValue - the base value that is being used. + * @param minFactor - range min - a 2 here makes us ensure that value + * valueTocheck is at least twice the baseValue. + * @param maxFactor - range max + * @return long + */ + private static long sanitizeUserArgs(long valueTocheck, long baseValue, + long minFactor, long maxFactor) + throws IllegalArgumentException { + if ((valueTocheck >= (baseValue * minFactor)) && + (valueTocheck <= (baseValue * maxFactor))) { + return valueTocheck; + } + String errMsg = String.format("%d is not within min = %d or max = " + + "%d", valueTocheck, baseValue * minFactor, baseValue * maxFactor); + throw new IllegalArgumentException(errMsg); + } + + + /** + * Returns the interval in which the heartbeat processor thread runs. + * + * @param conf - Configuration + * @return long in Milliseconds. + */ + public static long getScmheartbeatCheckerInterval(Configuration conf) { + return conf.getLong(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL_MS, + OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL_MS_DEFAULT); + } + + + /** + * Heartbeat Interval - Defines the heartbeat frequency from a datanode to + * SCM. + * + * @param conf - Ozone Config + * @return - HB interval in seconds. + */ + public static int getScmHeartbeatInterval(Configuration conf) { + return conf.getInt(OZONE_SCM_HEARTBEAT_INTERVAL_SECONDS, + OZONE_SCM_HEARBEAT_INTERVAL_SECONDS_DEFAULT); + } + + + /** + * Get the Stale Node interval, which is used by SCM to flag a datanode as + * stale, if the heartbeat from that node has been missing for this duration. + * + * @param conf - Configuration. + * @return - Long, Milliseconds to wait before flagging a node as stale. + */ + public static long getStaleNodeInterval(Configuration conf) { + + long staleNodeIntevalMs = conf.getLong(OZONE_SCM_STALENODE_INTERVAL_MS, + OZONE_SCM_STALENODE_INTERVAL_DEFAULT); + + long heartbeatThreadFrequencyMs = getScmheartbeatCheckerInterval(conf); + + long heartbeatIntervalMs = getScmHeartbeatInterval(conf) * 1000; + + + // Make sure that StaleNodeInterval is configured way above the frequency + // at which we run the heartbeat thread. + // + // Here we check that staleNodeInterval is at least five times more than the + // frequency at which the accounting thread is going to run. + try { + sanitizeUserArgs(staleNodeIntevalMs, heartbeatThreadFrequencyMs, 5, 1000); + } catch (IllegalArgumentException ex) { + LOG.error("Stale Node Interval MS is cannot be honored due to " + + "mis-configured {}. ex: {}", + OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL_MS, ex); + throw ex; + } + + // Make sure that stale node value is greater than configured value that + // datanodes are going to send HBs. + try { + sanitizeUserArgs(staleNodeIntevalMs, heartbeatIntervalMs, 3, 1000); + } catch (IllegalArgumentException ex) { + LOG.error("Stale Node Interval MS is cannot be honored due to " + + "mis-configured {}. ex: {}", + OZONE_SCM_HEARTBEAT_INTERVAL_SECONDS, ex); + throw ex; + } + return staleNodeIntevalMs; + } + + + /** + * Gets the interval for dead node flagging. This has to be a value that is + * greater than stale node value, and by transitive relation we also know + * that this value is greater than heartbeat interval and heartbeatProcess + * Interval. + * + * @param conf + * @return + */ + public static long getDeadNodeInterval(Configuration conf) { + long staleNodeIntervalMs = getStaleNodeInterval(conf); + long deadNodeIntervalMs = conf.getLong( + OZONE_SCM_DEADNODE_INTERVAL_MS, OZONE_SCM_DEADNODE_INTERVAL_DEFAULT); + + try { + // Make sure that dead nodes Ms is at least twice the time for staleNodes + // with a max of 1000 times the staleNodes. + sanitizeUserArgs(deadNodeIntervalMs, staleNodeIntervalMs, 2, 1000); + } catch (IllegalArgumentException ex) { + LOG.error("Dead Node Interval MS is cannot be honored due to " + + "mis-configured {}. ex: {}", + OZONE_SCM_STALENODE_INTERVAL_MS, ex); + throw ex; + } + return deadNodeIntervalMs; + } + + /** + * Returns the maximum number of heartbeat to process per loop of the process + * thread. + * @param conf Configration + * @return - int -- Number of HBs to process + */ + public static int getMaxHBToProcessPerLoop(Configuration conf){ + return conf.getInt(OZONE_SCM_MAX_HB_COUNT_TO_PROCESS, + OZONE_SCM_MAX_HB_COUNT_TO_PROCESS_DEFAULT); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java index 7c95fe7ccc0..ec133aa4b62 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java @@ -70,6 +70,31 @@ public final class OzoneConfigKeys { "ozone.scm.handler.count.key"; public static final int OZONE_SCM_HANDLER_COUNT_DEFAULT = 10; + public static final String OZONE_SCM_HEARTBEAT_INTERVAL_SECONDS = + "ozone.scm.heartbeat.interval.seconds"; + public static final int OZONE_SCM_HEARBEAT_INTERVAL_SECONDS_DEFAULT = + 30; + + public static final String OZONE_SCM_DEADNODE_INTERVAL_MS = + "ozone.scm.dead.node.interval.ms"; + public static final long OZONE_SCM_DEADNODE_INTERVAL_DEFAULT = + OZONE_SCM_HEARBEAT_INTERVAL_SECONDS_DEFAULT * 1000L * 20L; + + public static final String OZONE_SCM_MAX_HB_COUNT_TO_PROCESS = + "ozone.scm.max.hb.count.to.process"; + public static final int OZONE_SCM_MAX_HB_COUNT_TO_PROCESS_DEFAULT = 5000; + + public static final String OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL_MS = + "ozone.scm.heartbeat.thread.interval.ms"; + public static final long OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL_MS_DEFAULT = + 3000; + + public static final String OZONE_SCM_STALENODE_INTERVAL_MS = + "ozone.scm.stale.node.interval.ms"; + public static final long OZONE_SCM_STALENODE_INTERVAL_DEFAULT = + OZONE_SCM_HEARBEAT_INTERVAL_SECONDS_DEFAULT * 1000L * 3L; + + /** * There is no need to instantiate this class. */ diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/node/NodeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/node/NodeManager.java new file mode 100644 index 00000000000..699c789e5d7 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/node/NodeManager.java @@ -0,0 +1,120 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *
+ * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.ozone.scm.node; + +import org.apache.hadoop.hdfs.protocol.DatanodeID; +import org.apache.hadoop.hdfs.protocol.UnregisteredNodeException; +import org.apache.hadoop.hdfs.server.blockmanagement.UnresolvedTopologyException; + +import java.io.Closeable; +import java.util.List; + +/** + * A node manager supports a simple interface for managing a datanode. + * + * 1. A datanode registers with the NodeManager. + * + * 2. If the node is allowed to register, we add that to the nodes that we need + * to keep track of. + * + * 3. A heartbeat is made by the node at a fixed frequency. + * + * 4. A node can be in any of these 4 states: {HEALTHY, STALE, DEAD, + * DECOMMISSIONED} + * + * HEALTHY - It is a datanode that is regularly heartbeating us. + * + * STALE - A datanode for which we have missed few heart beats. + * + * DEAD - A datanode that we have not heard from for a while. + * + * DECOMMISSIONED - Someone told us to remove this node from the tracking + * list, by calling removeNode. We will throw away this nodes info soon. + */ +public interface NodeManager extends Closeable, Runnable { + + /** + * Update the heartbeat timestamp. + * + * @param datanodeID - Name of the datanode that send us heatbeat. + */ + void updateHeartbeat(DatanodeID datanodeID); + + /** + * Add a New Datanode to the NodeManager. + * + * @param nodeReg - Datanode ID. + * @throws UnresolvedTopologyException + */ + void registerNode(DatanodeID nodeReg) + throws UnresolvedTopologyException; + + /** + * Removes a data node from the management of this Node Manager. + * + * @param node - DataNode. + * @throws UnregisteredNodeException + */ + void removeNode(DatanodeID node) throws UnregisteredNodeException; + + /** + * Gets all Live Datanodes that is currently communicating with SCM. + * + * @return List of Datanodes that are Heartbeating SCM. + */ + + List+ * http://www.apache.org/licenses/LICENSE-2.0 + *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.ozone.scm.node;
+
+import org.apache.commons.lang.RandomStringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.protocol.DatanodeID;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.OzoneConfiguration;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.hamcrest.CoreMatchers;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+import java.io.IOException;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Random;
+import java.util.UUID;
+import java.util.concurrent.TimeoutException;
+
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SCM_DEADNODE_INTERVAL_MS;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SCM_HEARTBEAT_INTERVAL_SECONDS;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL_MS;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SCM_MAX_HB_COUNT_TO_PROCESS;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SCM_STALENODE_INTERVAL_MS;
+import static org.apache.hadoop.ozone.scm.node.NodeManager.NODESTATE.HEALTHY;
+import static org.apache.hadoop.ozone.scm.node.NodeManager.NODESTATE.STALE;
+import static org.apache.hadoop.ozone.scm.node.NodeManager.NODESTATE.DEAD;
+import static org.hamcrest.CoreMatchers.containsString;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.core.StringStartsWith.startsWith;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Test the Node Manager class.
+ */
+public class TestNodeManager {
+ @Rule
+ public ExpectedException thrown = ExpectedException.none();
+
+ @BeforeClass
+ public static void init() throws IOException {
+ }
+
+ /**
+ * Returns a new copy of Configuration.
+ *
+ * @return Config
+ */
+ Configuration getConf() {
+ return new OzoneConfiguration();
+ }
+
+ /**
+ * Create a new datanode ID.
+ *
+ * @return DatanodeID
+ */
+ DatanodeID getDatanodeID() {
+ return getDatanodeID(UUID.randomUUID().toString());
+ }
+
+ /**
+ * Create a new DatanodeID with NodeID set to the string.
+ *
+ * @param uuid - node ID, it is generally UUID.
+ * @return DatanodeID.
+ */
+ DatanodeID getDatanodeID(String uuid) {
+ Random random = new Random();
+ String ipAddress = random.nextInt(256) + "."
+ + random.nextInt(256) + "."
+ + random.nextInt(256) + "."
+ + random.nextInt(256);
+
+ String hostName = RandomStringUtils.randomAscii(8);
+ return new DatanodeID(ipAddress, hostName, uuid,
+ 0, 0, 0, 0);
+ }
+
+ /**
+ * Creates a NodeManager.
+ *
+ * @param config - Config for the node manager.
+ * @return SCNNodeManager
+ * @throws IOException
+ */
+
+ SCMNodeManager createNodeManager(Configuration config) throws IOException {
+ SCMNodeManager nodeManager = new SCMNodeManager(config);
+ assertFalse("Node manager should be in safe mode",
+ nodeManager.isOutOfNodeSafeMode());
+ return nodeManager;
+ }
+
+ /**
+ * Tests that Node manager handles heartbeats correctly, and comes out of Safe
+ * Mode.
+ *
+ * @throws IOException
+ * @throws InterruptedException
+ * @throws TimeoutException
+ */
+ @Test
+ public void testScmHeartbeat() throws IOException,
+ InterruptedException, TimeoutException {
+
+ try (SCMNodeManager nodeManager = createNodeManager(getConf())) {
+
+ // Send some heartbeats from different nodes.
+ for (int x = 0; x < nodeManager.getMinimumSafeModeNodes(); x++) {
+ nodeManager.updateHeartbeat(getDatanodeID());
+ }
+
+ // Wait for 4 seconds max.
+ GenericTestUtils.waitFor(() -> nodeManager.waitForHeartbeatThead(), 100,
+ 4 * 1000);
+
+ assertTrue("Heartbeat thread should have picked up the scheduled " +
+ "heartbeats and transitioned out of safe mode.",
+ nodeManager.isOutOfNodeSafeMode());
+ }
+ }
+
+ /**
+ * asserts that if we send no heartbeats node manager stays in safemode.
+ *
+ * @throws IOException
+ * @throws InterruptedException
+ * @throws TimeoutException
+ */
+ @Test
+ public void testScmNoHeartbeats() throws IOException,
+ InterruptedException, TimeoutException {
+
+ try (SCMNodeManager nodeManager = createNodeManager(getConf())) {
+ GenericTestUtils.waitFor(() -> nodeManager.waitForHeartbeatThead(), 100,
+ 4 * 1000);
+ assertFalse("No heartbeats, Node manager should have been in safe mode.",
+ nodeManager.isOutOfNodeSafeMode());
+ }
+ }
+
+ /**
+ * Asserts that if we don't get enough unique nodes we stay in safemode.
+ *
+ * @throws IOException
+ * @throws InterruptedException
+ * @throws TimeoutException
+ */
+ @Test
+ public void testScmNotEnoughHeartbeats() throws IOException,
+ InterruptedException, TimeoutException {
+ try (SCMNodeManager nodeManager = createNodeManager(getConf())) {
+
+ // Need 100 nodes to come out of safe mode, only one node is sending HB.
+ nodeManager.setMinimumSafeModeNodes(100);
+ nodeManager.updateHeartbeat(getDatanodeID());
+ GenericTestUtils.waitFor(() -> nodeManager.waitForHeartbeatThead(), 100,
+ 4 * 1000);
+ assertFalse("Not enough heartbeat, Node manager should have been in " +
+ "safemode.", nodeManager.isOutOfNodeSafeMode());
+ }
+ }
+
+ /**
+ * Asserts that many heartbeat from the same node is counted as a single
+ * node.
+ *
+ * @throws IOException
+ * @throws InterruptedException
+ * @throws TimeoutException
+ */
+ @Test
+ public void testScmSameNodeHeartbeats() throws IOException,
+ InterruptedException, TimeoutException {
+
+ try (SCMNodeManager nodeManager = createNodeManager(getConf())) {
+ nodeManager.setMinimumSafeModeNodes(3);
+ DatanodeID datanodeID = getDatanodeID();
+
+ // Send 10 heartbeat from same node, and assert we never leave safe mode.
+ for (int x = 0; x < 10; x++) {
+ nodeManager.updateHeartbeat(datanodeID);
+ }
+
+ GenericTestUtils.waitFor(() -> nodeManager.waitForHeartbeatThead(), 100,
+ 4 * 1000);
+ assertFalse("Not enough nodes have send heartbeat to node manager.",
+ nodeManager.isOutOfNodeSafeMode());
+ }
+ }
+
+ /**
+ * Asserts that adding heartbeats after shutdown does not work. This implies
+ * that heartbeat thread has been shutdown safely by closing the node
+ * manager.
+ *
+ * @throws IOException
+ * @throws InterruptedException
+ * @throws TimeoutException
+ */
+ @Test
+ public void testScmShutdown() throws IOException, InterruptedException,
+ TimeoutException {
+ Configuration conf = getConf();
+ conf.setInt(OzoneConfigKeys.OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL_MS, 100);
+ SCMNodeManager nodeManager = createNodeManager(conf);
+ nodeManager.close();
+
+ // These should never be processed.
+ nodeManager.updateHeartbeat(getDatanodeID());
+
+ // Let us just wait for 2 seconds to prove that HBs are not processed.
+ Thread.sleep(2 * 1000);
+
+ assertFalse("Node manager executor service is shutdown, should never exit" +
+ " safe mode", nodeManager.isOutOfNodeSafeMode());
+
+ assertEquals("Assert new HBs were never processed", 0,
+ nodeManager.getLastHBProcessedCount());
+ }
+
+ /**
+ * Asserts that we detect as many healthy nodes as we have generated heartbeat
+ * for.
+ *
+ * @throws IOException
+ * @throws InterruptedException
+ * @throws TimeoutException
+ */
+ @Test
+ public void testScmHealthyNodeCount() throws IOException,
+ InterruptedException, TimeoutException {
+ Configuration conf = getConf();
+ final int count = 10;
+
+ try (SCMNodeManager nodeManager = createNodeManager(conf)) {
+ for (int x = 0; x < count; x++) {
+ nodeManager.updateHeartbeat(getDatanodeID());
+ }
+ GenericTestUtils.waitFor(() -> nodeManager.waitForHeartbeatThead(), 100,
+ 4 * 1000);
+ assertEquals(count, nodeManager.getNodeCount(HEALTHY));
+ }
+ }
+
+ /**
+ * Asserts that if user provides a value less than 5 times the heartbeat
+ * interval as the StaleNode Value, we throw since that is a QoS that we
+ * cannot maintain.
+ *
+ * @throws IOException
+ * @throws InterruptedException
+ * @throws TimeoutException
+ */
+
+ @Test
+ public void testScmSanityOfUserConfig1() throws IOException,
+ InterruptedException, TimeoutException {
+ Configuration conf = getConf();
+ final int interval = 100;
+ conf.setInt(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL_MS, interval);
+ conf.setInt(OZONE_SCM_HEARTBEAT_INTERVAL_SECONDS, 1);
+
+ // This should be 5 times more than OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL_MS
+ // and 3 times more than OZONE_SCM_HEARTBEAT_INTERVAL_SECONDS
+ conf.setInt(OZONE_SCM_STALENODE_INTERVAL_MS, interval);
+
+ thrown.expect(IllegalArgumentException.class);
+
+ // This string is a multiple of the interval value
+ thrown.expectMessage(
+ startsWith("100 is not within min = 500 or max = 100000"));
+ createNodeManager(conf);
+ }
+
+ /**
+ * Asserts that if Stale Interval value is more than 5 times the value of HB
+ * processing thread it is a sane value.
+ *
+ * @throws IOException
+ * @throws InterruptedException
+ * @throws TimeoutException
+ */
+ @Test
+ public void testScmSanityOfUserConfig2() throws IOException,
+ InterruptedException, TimeoutException {
+ Configuration conf = getConf();
+ final int interval = 100;
+ conf.setInt(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL_MS, interval);
+ conf.setInt(OZONE_SCM_HEARTBEAT_INTERVAL_SECONDS, 1);
+
+ // This should be 5 times more than OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL_MS
+ // and 3 times more than OZONE_SCM_HEARTBEAT_INTERVAL_SECONDS
+ conf.setInt(OZONE_SCM_STALENODE_INTERVAL_MS, 3 * 1000);
+ createNodeManager(conf).close();
+ }
+
+ /**
+ * Asserts that a single node moves from Healthy to stale node if it misses
+ * the heartbeat.
+ *
+ * @throws IOException
+ * @throws InterruptedException
+ * @throws TimeoutException
+ */
+ @Test
+ public void testScmDetectStaleNode() throws IOException,
+ InterruptedException, TimeoutException {
+ Configuration conf = getConf();
+ final int interval = 100;
+ final int nodeCount = 10;
+ conf.setInt(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL_MS, interval);
+ conf.setInt(OZONE_SCM_HEARTBEAT_INTERVAL_SECONDS, 1);
+ // This should be 5 times more than OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL_MS
+ // and 3 times more than OZONE_SCM_HEARTBEAT_INTERVAL_SECONDS
+ conf.setInt(OZONE_SCM_STALENODE_INTERVAL_MS, 3 * 1000);
+
+ try (SCMNodeManager nodeManager = createNodeManager(conf)) {
+ List