HDDS-858. Start a Standalone Ratis Server on OM
This commit is contained in:
parent
8d882c3786
commit
019836b113
|
@ -1412,7 +1412,103 @@
|
||||||
<value>1MB</value>
|
<value>1MB</value>
|
||||||
<tag>OZONE, CLIENT, MANAGEMENT</tag>
|
<tag>OZONE, CLIENT, MANAGEMENT</tag>
|
||||||
<description>Checksum will be computed for every bytes per checksum number
|
<description>Checksum will be computed for every bytes per checksum number
|
||||||
of bytes and stored sequentially.
|
of bytes and stored sequentially. The minimum value for this config is
|
||||||
|
256KB.
|
||||||
|
</description>
|
||||||
|
</property>
|
||||||
|
|
||||||
|
<property>
|
||||||
|
<name>ozone.om.ratis.enable</name>
|
||||||
|
<value>false</value>
|
||||||
|
<tag>OZONE, OM, RATIS, MANAGEMENT</tag>
|
||||||
|
<description>Property to enable or disable Ratis server on OM.
|
||||||
|
Please note - this is a temporary property to disable OM Ratis server.
|
||||||
|
</description>
|
||||||
|
</property>
|
||||||
|
|
||||||
|
<property>
|
||||||
|
<name>ozone.om.ratis.port</name>
|
||||||
|
<value>9872</value>
|
||||||
|
<tag>OZONE, OM, RATIS</tag>
|
||||||
|
<description>
|
||||||
|
The port number of the OzoneManager's Ratis server.
|
||||||
|
</description>
|
||||||
|
</property>
|
||||||
|
|
||||||
|
<property>
|
||||||
|
<name>ozone.om.ratis.random.port</name>
|
||||||
|
<value>false</value>
|
||||||
|
<tag>OZONE, OM, RATIS, DEBUG</tag>
|
||||||
|
<description>Allocates a random free port for OM's Ratis server. This is
|
||||||
|
used only while running unit tests.
|
||||||
|
</description>
|
||||||
|
</property>
|
||||||
|
|
||||||
|
<property>
|
||||||
|
<name>ozone.om.ratis.rpc.type</name>
|
||||||
|
<value>GRPC</value>
|
||||||
|
<tag>OZONE, OM, RATIS, MANAGEMENT</tag>
|
||||||
|
<description>Ratis supports different kinds of transports like netty, GRPC,
|
||||||
|
Hadoop RPC etc. This picks one of those for this cluster.
|
||||||
|
</description>
|
||||||
|
</property>
|
||||||
|
|
||||||
|
<property>
|
||||||
|
<name>ozone.om.ratis.storage.dir</name>
|
||||||
|
<value/>
|
||||||
|
<tag>OZONE, OM, STORAGE, MANAGEMENT, RATIS</tag>
|
||||||
|
<description>This directory is used for storing OM's Ratis metadata like
|
||||||
|
logs. If this is not set then default metadata dirs is used. A warning
|
||||||
|
will be logged if this not set. Ideally, this should be mapped to a
|
||||||
|
fast disk like an SSD.
|
||||||
|
</description>
|
||||||
|
</property>
|
||||||
|
|
||||||
|
<property>
|
||||||
|
<name>ozone.om.ratis.segment.size</name>
|
||||||
|
<value>16KB</value>
|
||||||
|
<tag>OZONE, OM, RATIS, PERFORMANCE</tag>
|
||||||
|
<description>The size of the raft segment used by Apache Ratis on OM.
|
||||||
|
(16 KB by default)
|
||||||
|
</description>
|
||||||
|
</property>
|
||||||
|
|
||||||
|
<property>
|
||||||
|
<name>ozone.om.ratis.segment.preallocated.size</name>
|
||||||
|
<value>128MB</value>
|
||||||
|
<tag>OZONE, OM, RATIS, PERFORMANCE</tag>
|
||||||
|
<description>The size of the buffer which is preallocated for raft segment
|
||||||
|
used by Apache Ratis on OM.(128 MB by default)
|
||||||
|
</description>
|
||||||
|
</property>
|
||||||
|
|
||||||
|
<property>
|
||||||
|
<name>ozone.om.ratis.server.request.timeout</name>
|
||||||
|
<value>3s</value>
|
||||||
|
<tag>OZONE, OM, RATIS, MANAGEMENT</tag>
|
||||||
|
<description>The timeout duration for OM's ratis server request .</description>
|
||||||
|
</property>
|
||||||
|
|
||||||
|
<property>
|
||||||
|
<name>ozone.om.ratis.server.retry.cache.timeout</name>
|
||||||
|
<value>600000ms</value>
|
||||||
|
<tag>OZONE, OM, RATIS, MANAGEMENT</tag>
|
||||||
|
<description>Retry Cache entry timeout for OM's ratis server.</description>
|
||||||
|
</property>
|
||||||
|
|
||||||
|
<property>
|
||||||
|
<name>ozone.om.ratis.minimum.timeout</name>
|
||||||
|
<value>1s</value>
|
||||||
|
<tag>OZONE, OM, RATIS, MANAGEMENT</tag>
|
||||||
|
<description>The minimum timeout duration for OM's Ratis server rpc.
|
||||||
|
</description>
|
||||||
|
</property>
|
||||||
|
|
||||||
|
<property>
|
||||||
|
<name>ozone.om.ratis.client.request.timeout</name>
|
||||||
|
<value>3s</value>
|
||||||
|
<tag>OZONE, OM, RATIS, MANAGEMENT</tag>
|
||||||
|
<description>The timeout duration for OM Ratis client request.
|
||||||
</description>
|
</description>
|
||||||
</property>
|
</property>
|
||||||
|
|
||||||
|
|
|
@ -337,13 +337,16 @@ public final class HddsServerUtil {
|
||||||
OzoneConfigKeys.DFS_CONTAINER_RATIS_DATANODE_STORAGE_DIR);
|
OzoneConfigKeys.DFS_CONTAINER_RATIS_DATANODE_STORAGE_DIR);
|
||||||
|
|
||||||
if (Strings.isNullOrEmpty(storageDir)) {
|
if (Strings.isNullOrEmpty(storageDir)) {
|
||||||
LOG.warn("Storage directory for Ratis is not configured." +
|
storageDir = getDefaultRatisDirectory(conf);
|
||||||
"Mapping Ratis storage under {}. It is a good idea " +
|
|
||||||
"to map this to an SSD disk. Falling back to {}",
|
|
||||||
storageDir, HddsConfigKeys.OZONE_METADATA_DIRS);
|
|
||||||
File metaDirPath = ServerUtils.getOzoneMetaDirPath(conf);
|
|
||||||
storageDir = (new File (metaDirPath, "ratis")).getPath();
|
|
||||||
}
|
}
|
||||||
return storageDir;
|
return storageDir;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static String getDefaultRatisDirectory(Configuration conf) {
|
||||||
|
LOG.warn("Storage directory for Ratis is not configured. It is a good " +
|
||||||
|
"idea to map this to an SSD disk. Falling back to {}",
|
||||||
|
HddsConfigKeys.OZONE_METADATA_DIRS);
|
||||||
|
File metaDirPath = ServerUtils.getOzoneMetaDirPath(conf);
|
||||||
|
return (new File(metaDirPath, "ratis")).getPath();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -17,7 +17,10 @@
|
||||||
|
|
||||||
package org.apache.hadoop.ozone.om;
|
package org.apache.hadoop.ozone.om;
|
||||||
|
|
||||||
|
import java.util.concurrent.TimeUnit;
|
||||||
|
|
||||||
import org.apache.hadoop.ozone.OzoneAcl;
|
import org.apache.hadoop.ozone.OzoneAcl;
|
||||||
|
import org.apache.ratis.util.TimeDuration;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Ozone Manager Constants.
|
* Ozone Manager Constants.
|
||||||
|
@ -86,4 +89,61 @@ public final class OMConfigKeys {
|
||||||
public static final String OZONE_OM_METRICS_SAVE_INTERVAL =
|
public static final String OZONE_OM_METRICS_SAVE_INTERVAL =
|
||||||
"ozone.om.save.metrics.interval";
|
"ozone.om.save.metrics.interval";
|
||||||
public static final String OZONE_OM_METRICS_SAVE_INTERVAL_DEFAULT = "5m";
|
public static final String OZONE_OM_METRICS_SAVE_INTERVAL_DEFAULT = "5m";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* OM Ratis related configurations.
|
||||||
|
*/
|
||||||
|
public static final String OZONE_OM_RATIS_ENABLE_KEY
|
||||||
|
= "ozone.om.ratis.enable";
|
||||||
|
public static final boolean OZONE_OM_RATIS_ENABLE_DEFAULT
|
||||||
|
= false;
|
||||||
|
public static final String OZONE_OM_RATIS_PORT_KEY
|
||||||
|
= "ozone.om.ratis.port";
|
||||||
|
public static final int OZONE_OM_RATIS_PORT_DEFAULT
|
||||||
|
= 9872;
|
||||||
|
// When set to true, allocate a random free port for ozone ratis server
|
||||||
|
public static final String OZONE_OM_RATIS_RANDOM_PORT_KEY =
|
||||||
|
"ozone.om.ratis.random.port";
|
||||||
|
public static final boolean OZONE_OM_RATIS_RANDOM_PORT_KEY_DEFAULT
|
||||||
|
= false;
|
||||||
|
public static final String OZONE_OM_RATIS_RPC_TYPE_KEY
|
||||||
|
= "ozone.om.ratis.rpc.type";
|
||||||
|
public static final String OZONE_OM_RATIS_RPC_TYPE_DEFAULT
|
||||||
|
= "GRPC";
|
||||||
|
|
||||||
|
// OM Ratis Log configurations
|
||||||
|
public static final String OZONE_OM_RATIS_STORAGE_DIR
|
||||||
|
= "ozone.om.ratis.storage.dir";
|
||||||
|
public static final String OZONE_OM_RATIS_SEGMENT_SIZE_KEY
|
||||||
|
= "ozone.om.ratis.segment.size";
|
||||||
|
public static final String OZONE_OM_RATIS_SEGMENT_SIZE_DEFAULT
|
||||||
|
= "16KB";
|
||||||
|
public static final String OZONE_OM_RATIS_SEGMENT_PREALLOCATED_SIZE_KEY
|
||||||
|
= "ozone.om.ratis.segment.preallocated.size";
|
||||||
|
public static final String OZONE_OM_RATIS_SEGMENT_PREALLOCATED_SIZE_DEFAULT
|
||||||
|
= "128MB";
|
||||||
|
|
||||||
|
// OM Ratis server configurations
|
||||||
|
public static final String OZONE_OM_RATIS_SERVER_REQUEST_TIMEOUT_KEY
|
||||||
|
= "ozone.om.ratis.server.request.timeout";
|
||||||
|
public static final TimeDuration
|
||||||
|
OZONE_OM_RATIS_SERVER_REQUEST_TIMEOUT_DEFAULT
|
||||||
|
= TimeDuration.valueOf(3000, TimeUnit.MILLISECONDS);
|
||||||
|
public static final String
|
||||||
|
OZONE_OM_RATIS_SERVER_RETRY_CACHE_TIMEOUT_KEY
|
||||||
|
= "ozone.om.ratis.server.retry.cache.timeout";
|
||||||
|
public static final TimeDuration
|
||||||
|
OZONE_OM_RATIS_SERVER_RETRY_CACHE_TIMEOUT_DEFAULT
|
||||||
|
= TimeDuration.valueOf(600000, TimeUnit.MILLISECONDS);
|
||||||
|
public static final String OZONE_OM_RATIS_MINIMUM_TIMEOUT_KEY
|
||||||
|
= "ozone.om.ratis.minimum.timeout";
|
||||||
|
public static final TimeDuration OZONE_OM_RATIS_MINIMUM_TIMEOUT_DEFAULT
|
||||||
|
= TimeDuration.valueOf(1, TimeUnit.SECONDS);
|
||||||
|
|
||||||
|
// OM Ratis client configurations
|
||||||
|
public static final String OZONE_OM_RATIS_CLIENT_REQUEST_TIMEOUT_KEY
|
||||||
|
= "ozone.om.ratis.client.request.timeout";
|
||||||
|
public static final TimeDuration
|
||||||
|
OZONE_OM_RATIS_CLIENT_REQUEST_TIMEOUT_DEFAULT
|
||||||
|
= TimeDuration.valueOf(3000, TimeUnit.MILLISECONDS);
|
||||||
}
|
}
|
||||||
|
|
|
@ -62,6 +62,7 @@ import org.apache.hadoop.util.Time;
|
||||||
import org.apache.hadoop.utils.db.Table;
|
import org.apache.hadoop.utils.db.Table;
|
||||||
import org.apache.hadoop.utils.db.Table.KeyValue;
|
import org.apache.hadoop.utils.db.Table.KeyValue;
|
||||||
import org.apache.hadoop.utils.db.TableIterator;
|
import org.apache.hadoop.utils.db.TableIterator;
|
||||||
|
import org.apache.ratis.util.LifeCycle;
|
||||||
import org.junit.AfterClass;
|
import org.junit.AfterClass;
|
||||||
import org.junit.Assert;
|
import org.junit.Assert;
|
||||||
import org.junit.BeforeClass;
|
import org.junit.BeforeClass;
|
||||||
|
@ -1368,4 +1369,25 @@ public class TestOzoneManager {
|
||||||
Assert.assertEquals(NetUtils.createSocketAddr(
|
Assert.assertEquals(NetUtils.createSocketAddr(
|
||||||
conf.get(OZONE_SCM_CLIENT_ADDRESS_KEY)), scmAddress);
|
conf.get(OZONE_SCM_CLIENT_ADDRESS_KEY)), scmAddress);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Test that OM Ratis server is started only when OZONE_OM_RATIS_ENABLE_KEY is
|
||||||
|
* set to true.
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
public void testRatsiServerOnOmInitialization() throws IOException {
|
||||||
|
// OM Ratis server should not be started when OZONE_OM_RATIS_ENABLE_KEY
|
||||||
|
// is not set to true
|
||||||
|
Assert.assertNull("OM Ratis server started though OM Ratis is disabled.",
|
||||||
|
cluster.getOzoneManager().getOmRatisServerState());
|
||||||
|
|
||||||
|
// Enable OM Ratis and restart OM
|
||||||
|
conf.setBoolean(OMConfigKeys.OZONE_OM_RATIS_ENABLE_KEY, true);
|
||||||
|
cluster.restartOzoneManager();
|
||||||
|
|
||||||
|
// On enabling OM Ratis, the Ratis server should be started
|
||||||
|
Assert.assertEquals("OM Ratis server did not start",
|
||||||
|
LifeCycle.State.RUNNING,
|
||||||
|
cluster.getOzoneManager().getOmRatisServerState());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -66,6 +66,7 @@ import org.apache.hadoop.ozone.om.helpers.OpenKeySession;
|
||||||
import org.apache.hadoop.ozone.om.helpers.ServiceInfo;
|
import org.apache.hadoop.ozone.om.helpers.ServiceInfo;
|
||||||
import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol;
|
import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol;
|
||||||
import org.apache.hadoop.ozone.om.protocolPB.OzoneManagerProtocolPB;
|
import org.apache.hadoop.ozone.om.protocolPB.OzoneManagerProtocolPB;
|
||||||
|
import org.apache.hadoop.ozone.om.ratis.OzoneManagerRatisServer;
|
||||||
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OzoneAclInfo;
|
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OzoneAclInfo;
|
||||||
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ServicePort;
|
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ServicePort;
|
||||||
import org.apache.hadoop.ozone.protocolPB.OzoneManagerProtocolServerSideTranslatorPB;
|
import org.apache.hadoop.ozone.protocolPB.OzoneManagerProtocolServerSideTranslatorPB;
|
||||||
|
@ -73,6 +74,7 @@ import org.apache.hadoop.security.UserGroupInformation;
|
||||||
import org.apache.hadoop.util.GenericOptionsParser;
|
import org.apache.hadoop.util.GenericOptionsParser;
|
||||||
import org.apache.hadoop.util.ShutdownHookManager;
|
import org.apache.hadoop.util.ShutdownHookManager;
|
||||||
import org.apache.hadoop.util.StringUtils;
|
import org.apache.hadoop.util.StringUtils;
|
||||||
|
import org.apache.ratis.util.LifeCycle;
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
@ -136,6 +138,7 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
|
||||||
private final OzoneConfiguration configuration;
|
private final OzoneConfiguration configuration;
|
||||||
private RPC.Server omRpcServer;
|
private RPC.Server omRpcServer;
|
||||||
private InetSocketAddress omRpcAddress;
|
private InetSocketAddress omRpcAddress;
|
||||||
|
private OzoneManagerRatisServer omRatisServer;
|
||||||
private final OMMetadataManager metadataManager;
|
private final OMMetadataManager metadataManager;
|
||||||
private final VolumeManager volumeManager;
|
private final VolumeManager volumeManager;
|
||||||
private final BucketManager bucketManager;
|
private final BucketManager bucketManager;
|
||||||
|
@ -509,6 +512,15 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
|
||||||
return omStorage;
|
return omStorage;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@VisibleForTesting
|
||||||
|
public LifeCycle.State getOmRatisServerState() {
|
||||||
|
if (omRatisServer == null) {
|
||||||
|
return null;
|
||||||
|
} else {
|
||||||
|
return omRatisServer.getServerState();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get metadata manager.
|
* Get metadata manager.
|
||||||
*
|
*
|
||||||
|
@ -542,6 +554,22 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
|
||||||
LOG.info(buildRpcServerStartMessage("OzoneManager RPC server",
|
LOG.info(buildRpcServerStartMessage("OzoneManager RPC server",
|
||||||
omRpcAddress));
|
omRpcAddress));
|
||||||
|
|
||||||
|
boolean omRatisEnabled = configuration.getBoolean(
|
||||||
|
OMConfigKeys.OZONE_OM_RATIS_ENABLE_KEY,
|
||||||
|
OMConfigKeys.OZONE_OM_RATIS_ENABLE_DEFAULT);
|
||||||
|
// This is a temporary check. Once fully implemented, all OM state change
|
||||||
|
// should go through Ratis, either standalone (for non-HA) or replicated
|
||||||
|
// (for HA).
|
||||||
|
if (omRatisEnabled) {
|
||||||
|
omRatisServer = OzoneManagerRatisServer.newOMRatisServer(
|
||||||
|
omStorage.getOmId(), configuration);
|
||||||
|
omRatisServer.start();
|
||||||
|
|
||||||
|
LOG.info("OzoneManager Ratis server started at port {}",
|
||||||
|
omRatisServer.getServerPort());
|
||||||
|
} else {
|
||||||
|
omRatisServer = null;
|
||||||
|
}
|
||||||
|
|
||||||
DefaultMetricsSystem.initialize("OzoneManager");
|
DefaultMetricsSystem.initialize("OzoneManager");
|
||||||
|
|
||||||
|
@ -584,6 +612,9 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
|
||||||
metricsTimer = null;
|
metricsTimer = null;
|
||||||
scheduleOMMetricsWriteTask = null;
|
scheduleOMMetricsWriteTask = null;
|
||||||
omRpcServer.stop();
|
omRpcServer.stop();
|
||||||
|
if (omRatisServer != null) {
|
||||||
|
omRatisServer.stop();
|
||||||
|
}
|
||||||
keyManager.stop();
|
keyManager.stop();
|
||||||
httpServer.stop();
|
httpServer.stop();
|
||||||
metadataManager.stop();
|
metadataManager.stop();
|
||||||
|
|
|
@ -0,0 +1,271 @@
|
||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.apache.hadoop.ozone.om.ratis;
|
||||||
|
|
||||||
|
import com.google.common.annotations.VisibleForTesting;
|
||||||
|
import com.google.common.base.Strings;
|
||||||
|
import java.io.File;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.net.InetSocketAddress;
|
||||||
|
import java.net.ServerSocket;
|
||||||
|
import java.net.SocketAddress;
|
||||||
|
import java.util.Collections;
|
||||||
|
import java.util.Objects;
|
||||||
|
import java.util.concurrent.TimeUnit;
|
||||||
|
|
||||||
|
import org.apache.hadoop.conf.Configuration;
|
||||||
|
import org.apache.hadoop.conf.StorageUnit;
|
||||||
|
import org.apache.hadoop.hdds.scm.HddsServerUtil;
|
||||||
|
import org.apache.hadoop.ozone.om.OMConfigKeys;
|
||||||
|
import org.apache.ratis.RaftConfigKeys;
|
||||||
|
import org.apache.ratis.client.RaftClientConfigKeys;
|
||||||
|
import org.apache.ratis.conf.RaftProperties;
|
||||||
|
import org.apache.ratis.grpc.GrpcConfigKeys;
|
||||||
|
import org.apache.ratis.netty.NettyConfigKeys;
|
||||||
|
import org.apache.ratis.protocol.RaftGroupId;
|
||||||
|
import org.apache.ratis.protocol.RaftPeerId;
|
||||||
|
import org.apache.ratis.rpc.RpcType;
|
||||||
|
import org.apache.ratis.rpc.SupportedRpcType;
|
||||||
|
import org.apache.ratis.server.RaftServer;
|
||||||
|
import org.apache.ratis.server.RaftServerConfigKeys;
|
||||||
|
import org.apache.ratis.statemachine.impl.BaseStateMachine;
|
||||||
|
import org.apache.ratis.util.LifeCycle;
|
||||||
|
import org.apache.ratis.util.SizeInBytes;
|
||||||
|
import org.apache.ratis.util.TimeDuration;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Creates a Ratis server endpoint for OM.
|
||||||
|
*/
|
||||||
|
public final class OzoneManagerRatisServer {
|
||||||
|
private static final Logger LOG = LoggerFactory
|
||||||
|
.getLogger(OzoneManagerRatisServer.class);
|
||||||
|
|
||||||
|
private final int port;
|
||||||
|
private final RaftServer server;
|
||||||
|
|
||||||
|
private OzoneManagerRatisServer(String omId, int port, Configuration conf)
|
||||||
|
throws IOException {
|
||||||
|
Objects.requireNonNull(omId, "omId == null");
|
||||||
|
this.port = port;
|
||||||
|
RaftProperties serverProperties = newRaftProperties(conf);
|
||||||
|
|
||||||
|
this.server = RaftServer.newBuilder()
|
||||||
|
.setServerId(RaftPeerId.valueOf(omId))
|
||||||
|
.setProperties(serverProperties)
|
||||||
|
.setStateMachineRegistry(this::getStateMachine)
|
||||||
|
.build();
|
||||||
|
}
|
||||||
|
|
||||||
|
public static OzoneManagerRatisServer newOMRatisServer(String omId,
|
||||||
|
Configuration ozoneConf) throws IOException {
|
||||||
|
int localPort = ozoneConf.getInt(
|
||||||
|
OMConfigKeys.OZONE_OM_RATIS_PORT_KEY,
|
||||||
|
OMConfigKeys.OZONE_OM_RATIS_PORT_DEFAULT);
|
||||||
|
|
||||||
|
// Get an available port on current node and
|
||||||
|
// use that as the container port
|
||||||
|
if (ozoneConf.getBoolean(
|
||||||
|
OMConfigKeys.OZONE_OM_RATIS_RANDOM_PORT_KEY,
|
||||||
|
OMConfigKeys.OZONE_OM_RATIS_RANDOM_PORT_KEY_DEFAULT)) {
|
||||||
|
try (ServerSocket socket = new ServerSocket()) {
|
||||||
|
socket.setReuseAddress(true);
|
||||||
|
SocketAddress address = new InetSocketAddress(0);
|
||||||
|
socket.bind(address);
|
||||||
|
localPort = socket.getLocalPort();
|
||||||
|
LOG.info("Found a free port for the OM Ratis server : {}", localPort);
|
||||||
|
} catch (IOException e) {
|
||||||
|
LOG.error("Unable find a random free port for the server, "
|
||||||
|
+ "fallback to use default port {}", localPort, e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return new OzoneManagerRatisServer(omId, localPort, ozoneConf);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Return a dummy StateMachine.
|
||||||
|
* TODO: Implement a state machine on OM.
|
||||||
|
*/
|
||||||
|
private BaseStateMachine getStateMachine(RaftGroupId gid) {
|
||||||
|
return new BaseStateMachine();
|
||||||
|
}
|
||||||
|
|
||||||
|
public void start() throws IOException {
|
||||||
|
LOG.info("Starting {} {} at port {}", getClass().getSimpleName(),
|
||||||
|
server.getId(), port);
|
||||||
|
server.start();
|
||||||
|
}
|
||||||
|
|
||||||
|
public void stop() {
|
||||||
|
try {
|
||||||
|
server.close();
|
||||||
|
} catch (IOException e) {
|
||||||
|
throw new RuntimeException(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private RaftProperties newRaftProperties(Configuration conf) {
|
||||||
|
final RaftProperties properties = new RaftProperties();
|
||||||
|
|
||||||
|
// Set RPC type
|
||||||
|
final String rpcType = conf.get(
|
||||||
|
OMConfigKeys.OZONE_OM_RATIS_RPC_TYPE_KEY,
|
||||||
|
OMConfigKeys.OZONE_OM_RATIS_RPC_TYPE_DEFAULT);
|
||||||
|
final RpcType rpc = SupportedRpcType.valueOfIgnoreCase(rpcType);
|
||||||
|
RaftConfigKeys.Rpc.setType(properties, rpc);
|
||||||
|
|
||||||
|
// Set the ratis port number
|
||||||
|
if (rpc == SupportedRpcType.GRPC) {
|
||||||
|
GrpcConfigKeys.Server.setPort(properties, port);
|
||||||
|
} else if (rpc == SupportedRpcType.NETTY) {
|
||||||
|
NettyConfigKeys.Server.setPort(properties, port);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Set Ratis storage directory
|
||||||
|
String storageDir = getOMRatisDirectory(conf);
|
||||||
|
RaftServerConfigKeys.setStorageDirs(properties,
|
||||||
|
Collections.singletonList(new File(storageDir)));
|
||||||
|
|
||||||
|
// Set RAFT segment size
|
||||||
|
final int raftSegmentSize = (int) conf.getStorageSize(
|
||||||
|
OMConfigKeys.OZONE_OM_RATIS_SEGMENT_SIZE_KEY,
|
||||||
|
OMConfigKeys.OZONE_OM_RATIS_SEGMENT_SIZE_DEFAULT,
|
||||||
|
StorageUnit.BYTES);
|
||||||
|
RaftServerConfigKeys.Log.setSegmentSizeMax(properties,
|
||||||
|
SizeInBytes.valueOf(raftSegmentSize));
|
||||||
|
|
||||||
|
// Set RAFT segment pre-allocated size
|
||||||
|
final int raftSegmentPreallocatedSize = (int) conf.getStorageSize(
|
||||||
|
OMConfigKeys.OZONE_OM_RATIS_SEGMENT_PREALLOCATED_SIZE_KEY,
|
||||||
|
OMConfigKeys.OZONE_OM_RATIS_SEGMENT_PREALLOCATED_SIZE_DEFAULT,
|
||||||
|
StorageUnit.BYTES);
|
||||||
|
RaftServerConfigKeys.Log.Appender.setBufferCapacity(properties,
|
||||||
|
SizeInBytes.valueOf(raftSegmentPreallocatedSize));
|
||||||
|
RaftServerConfigKeys.Log.setPreallocatedSize(properties,
|
||||||
|
SizeInBytes.valueOf(raftSegmentPreallocatedSize));
|
||||||
|
|
||||||
|
// For grpc set the maximum message size
|
||||||
|
// TODO: calculate the max message size based on the max size of a
|
||||||
|
// PutSmallFileRequest's file size limit
|
||||||
|
GrpcConfigKeys.setMessageSizeMax(properties,
|
||||||
|
SizeInBytes.valueOf(raftSegmentPreallocatedSize));
|
||||||
|
|
||||||
|
// Set the server request timeout
|
||||||
|
TimeUnit serverRequestTimeoutUnit =
|
||||||
|
OMConfigKeys.OZONE_OM_RATIS_SERVER_REQUEST_TIMEOUT_DEFAULT.getUnit();
|
||||||
|
long serverRequestTimeoutDuration = conf.getTimeDuration(
|
||||||
|
OMConfigKeys.OZONE_OM_RATIS_SERVER_REQUEST_TIMEOUT_KEY,
|
||||||
|
OMConfigKeys.OZONE_OM_RATIS_SERVER_REQUEST_TIMEOUT_DEFAULT
|
||||||
|
.getDuration(), serverRequestTimeoutUnit);
|
||||||
|
final TimeDuration serverRequestTimeout = TimeDuration.valueOf(
|
||||||
|
serverRequestTimeoutDuration, serverRequestTimeoutUnit);
|
||||||
|
RaftServerConfigKeys.Rpc.setRequestTimeout(properties,
|
||||||
|
serverRequestTimeout);
|
||||||
|
|
||||||
|
// Set timeout for server retry cache entry
|
||||||
|
TimeUnit retryCacheTimeoutUnit = OMConfigKeys
|
||||||
|
.OZONE_OM_RATIS_SERVER_RETRY_CACHE_TIMEOUT_DEFAULT.getUnit();
|
||||||
|
long retryCacheTimeoutDuration = conf.getTimeDuration(
|
||||||
|
OMConfigKeys.OZONE_OM_RATIS_SERVER_RETRY_CACHE_TIMEOUT_KEY,
|
||||||
|
OMConfigKeys.OZONE_OM_RATIS_SERVER_RETRY_CACHE_TIMEOUT_DEFAULT
|
||||||
|
.getDuration(), retryCacheTimeoutUnit);
|
||||||
|
final TimeDuration retryCacheTimeout = TimeDuration.valueOf(
|
||||||
|
retryCacheTimeoutDuration, retryCacheTimeoutUnit);
|
||||||
|
RaftServerConfigKeys.RetryCache
|
||||||
|
.setExpiryTime(properties, retryCacheTimeout);
|
||||||
|
|
||||||
|
// Set the server min and max timeout
|
||||||
|
TimeUnit serverMinTimeoutUnit =
|
||||||
|
OMConfigKeys.OZONE_OM_RATIS_MINIMUM_TIMEOUT_DEFAULT.getUnit();
|
||||||
|
long serverMinTimeoutDuration = conf.getTimeDuration(
|
||||||
|
OMConfigKeys.OZONE_OM_RATIS_MINIMUM_TIMEOUT_KEY,
|
||||||
|
OMConfigKeys.OZONE_OM_RATIS_MINIMUM_TIMEOUT_DEFAULT
|
||||||
|
.getDuration(), serverMinTimeoutUnit);
|
||||||
|
final TimeDuration serverMinTimeout = TimeDuration.valueOf(
|
||||||
|
serverMinTimeoutDuration, serverMinTimeoutUnit);
|
||||||
|
long serverMaxTimeoutDuration =
|
||||||
|
serverMinTimeout.toLong(TimeUnit.MILLISECONDS) + 200;
|
||||||
|
final TimeDuration serverMaxTimeout = TimeDuration.valueOf(
|
||||||
|
serverMaxTimeoutDuration, serverMinTimeoutUnit);
|
||||||
|
RaftServerConfigKeys.Rpc.setTimeoutMin(properties,
|
||||||
|
serverMinTimeout);
|
||||||
|
RaftServerConfigKeys.Rpc.setTimeoutMax(properties,
|
||||||
|
serverMaxTimeout);
|
||||||
|
|
||||||
|
// Enable batch append on raft server
|
||||||
|
RaftServerConfigKeys.Log.Appender.setBatchEnabled(properties, true);
|
||||||
|
|
||||||
|
// Set the number of maximum cached segments
|
||||||
|
RaftServerConfigKeys.Log.setMaxCachedSegmentNum(properties, 2);
|
||||||
|
|
||||||
|
// Set the client request timeout
|
||||||
|
TimeUnit clientRequestTimeoutUnit =
|
||||||
|
OMConfigKeys.OZONE_OM_RATIS_CLIENT_REQUEST_TIMEOUT_DEFAULT.getUnit();
|
||||||
|
long clientRequestTimeoutDuration = conf.getTimeDuration(
|
||||||
|
OMConfigKeys.OZONE_OM_RATIS_CLIENT_REQUEST_TIMEOUT_KEY,
|
||||||
|
OMConfigKeys.OZONE_OM_RATIS_CLIENT_REQUEST_TIMEOUT_DEFAULT
|
||||||
|
.getDuration(), clientRequestTimeoutUnit);
|
||||||
|
final TimeDuration clientRequestTimeout = TimeDuration.valueOf(
|
||||||
|
clientRequestTimeoutDuration, clientRequestTimeoutUnit);
|
||||||
|
RaftClientConfigKeys.Rpc.setRequestTimeout(properties,
|
||||||
|
clientRequestTimeout);
|
||||||
|
|
||||||
|
// TODO: set max write buffer size
|
||||||
|
|
||||||
|
/**
|
||||||
|
* TODO: when state machine is implemented, enable StateMachineData sync
|
||||||
|
* and set sync timeout and number of sync retries.
|
||||||
|
*/
|
||||||
|
|
||||||
|
/**
|
||||||
|
* TODO: set following ratis leader election related configs when
|
||||||
|
* replicated ratis server is implemented.
|
||||||
|
* 1. leader election timeout
|
||||||
|
* 2. node failure timeout
|
||||||
|
* 3.
|
||||||
|
*/
|
||||||
|
|
||||||
|
/**
|
||||||
|
* TODO: when ratis snapshots are implemented, set snapshot threshold and
|
||||||
|
* queue size.
|
||||||
|
*/
|
||||||
|
|
||||||
|
return properties;
|
||||||
|
}
|
||||||
|
|
||||||
|
public int getServerPort() {
|
||||||
|
return port;
|
||||||
|
}
|
||||||
|
|
||||||
|
@VisibleForTesting
|
||||||
|
public LifeCycle.State getServerState() {
|
||||||
|
return server.getLifeCycleState();
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String getOMRatisDirectory(Configuration conf) {
|
||||||
|
String storageDir = conf.get(OMConfigKeys.OZONE_OM_RATIS_STORAGE_DIR);
|
||||||
|
|
||||||
|
if (Strings.isNullOrEmpty(storageDir)) {
|
||||||
|
storageDir = HddsServerUtil.getDefaultRatisDirectory(conf);
|
||||||
|
}
|
||||||
|
return storageDir;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -0,0 +1,22 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.hadoop.ozone.om.ratis;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This package contains classes for the OM Ratis server implementation.
|
||||||
|
*/
|
|
@ -0,0 +1,69 @@
|
||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.apache.hadoop.ozone.om.ratis;
|
||||||
|
|
||||||
|
import java.nio.file.Path;
|
||||||
|
import java.nio.file.Paths;
|
||||||
|
import java.util.UUID;
|
||||||
|
import org.apache.hadoop.conf.Configuration;
|
||||||
|
import org.apache.hadoop.hdds.HddsConfigKeys;
|
||||||
|
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
|
||||||
|
import org.apache.hadoop.test.GenericTestUtils;
|
||||||
|
import org.apache.ratis.util.LifeCycle;
|
||||||
|
import org.junit.After;
|
||||||
|
import org.junit.Assert;
|
||||||
|
import org.junit.Before;
|
||||||
|
import org.junit.Test;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Test OM Ratis server.
|
||||||
|
*/
|
||||||
|
public class TestOzoneManagerRatisServer {
|
||||||
|
|
||||||
|
private Configuration conf;
|
||||||
|
private OzoneManagerRatisServer omRatisServer;
|
||||||
|
private String omID;
|
||||||
|
|
||||||
|
@Before
|
||||||
|
public void init() {
|
||||||
|
conf = new OzoneConfiguration();
|
||||||
|
omID = UUID.randomUUID().toString();
|
||||||
|
final String path = GenericTestUtils.getTempPath(omID);
|
||||||
|
Path metaDirPath = Paths.get(path, "om-meta");
|
||||||
|
conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, metaDirPath.toString());
|
||||||
|
}
|
||||||
|
|
||||||
|
@After
|
||||||
|
public void shutdown() {
|
||||||
|
if (omRatisServer != null) {
|
||||||
|
omRatisServer.stop();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Start a OM Ratis Server and checks its state.
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
public void testStartOMRatisServer() throws Exception {
|
||||||
|
omRatisServer = OzoneManagerRatisServer.newOMRatisServer(omID, conf);
|
||||||
|
omRatisServer.start();
|
||||||
|
Assert.assertEquals("Ratis Server should be in running state",
|
||||||
|
LifeCycle.State.RUNNING, omRatisServer.getServerState());
|
||||||
|
}
|
||||||
|
}
|
Loading…
Reference in New Issue