Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-13099. RBF: Use the ZooKeeper as the default State Store. Contributed by Yiqun Lin.

This commit is contained in:
Yiqun Lin 2018-02-09 13:57:42 +08:00
parent ddec08d7cc
commit 543f3abbee
5 changed files with 28 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
View File

@ -34,8 +34,8 @@ import org.apache.hadoop.hdfs.server.federation.metrics.FederationRPCPerformance
import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver;
import org.apache.hadoop.hdfs.server.federation.resolver.MembershipNamenodeResolver;
import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver;
import org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreFileImpl;
import org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreSerializerPBImpl;
import org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreZooKeeperImpl;
import org.apache.hadoop.http.HttpConfig;
/**
@ -1275,7 +1275,7 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final String FEDERATION_STORE_DRIVER_CLASS =
FEDERATION_STORE_PREFIX + "driver.class";
public static final Class<? extends StateStoreDriver>
FEDERATION_STORE_DRIVER_CLASS_DEFAULT = StateStoreFileImpl.class;
FEDERATION_STORE_DRIVER_CLASS_DEFAULT = StateStoreZooKeeperImpl.class;
public static final String FEDERATION_STORE_CONNECTION_TEST_MS =
FEDERATION_STORE_PREFIX + "connection.test";

10
hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
View File

@ -5085,9 +5085,15 @@
<property>
<name>dfs.federation.router.store.driver.class</name>
<value>org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreFileImpl</value>
<value>org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreZooKeeperImpl</value>
<description>
Class to implement the State Store. By default it uses the local disk.
Class to implement the State Store. There are three implementation classes currently
being supported:
org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreFileImpl,
org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreFileSystemImpl and
org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreZooKeeperImpl.
These implementation classes use the local file, filesystem and ZooKeeper as a backend respectively.
By default it uses the ZooKeeper as the default State Store.
</description>
</property>

2
hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSRouterFederation.md
View File

@ -325,7 +325,7 @@ The connection to the State Store and the internal caching at the Router.
|:---- |:---- |:---- |
| dfs.federation.router.store.enable | `true` | If `true`, the Router connects to the State Store. |
| dfs.federation.router.store.serializer | `StateStoreSerializerPBImpl` | Class to serialize State Store records. |
| dfs.federation.router.store.driver.class | `StateStoreZKImpl` | Class to implement the State Store. |
| dfs.federation.router.store.driver.class | `StateStoreZooKeeperImpl` | Class to implement the State Store. |
| dfs.federation.router.store.connection.test | 60000 | How often to check for the connection to the State Store in milliseconds. |
| dfs.federation.router.cache.ttl | 60000 | How often to refresh the State Store caches in milliseconds. |
| dfs.federation.router.store.membership.expiration | 300000 | Expiration time in milliseconds for a membership record. |

6
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterConfigBuilder.java
View File

@ -19,6 +19,8 @@ package org.apache.hadoop.hdfs.server.federation;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils;
import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver;
/**
* Constructs a router configuration with individual features enabled/disabled.
@ -119,6 +121,10 @@ public class RouterConfigBuilder {
}
public RouterConfigBuilder stateStore() {
// reset the State Store driver implementation class for testing
conf.setClass(DFSConfigKeys.FEDERATION_STORE_DRIVER_CLASS,
FederationStateStoreTestUtils.getTestDriverClass(),
StateStoreDriver.class);
return this.stateStore(true);
}

17
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/store/FederationStateStoreTestUtils.java
View File

@ -39,6 +39,7 @@ import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState;
import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver;
import org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreFileBaseImpl;
import org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreFileImpl;
import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
import org.apache.hadoop.hdfs.server.federation.store.records.MembershipState;
import org.apache.hadoop.hdfs.server.federation.store.records.MembershipStats;
@ -50,17 +51,21 @@ import org.apache.hadoop.util.Time;
*/
public final class FederationStateStoreTestUtils {
/** The State Store Driver implementation class for testing .*/
private static final Class<? extends StateStoreDriver>
FEDERATION_STORE_DRIVER_CLASS_FOR_TEST = StateStoreFileImpl.class;
private FederationStateStoreTestUtils() {
// Utility Class
}
/**
* Get the default State Store driver implementation.
* Get the State Store driver implementation for testing.
*
* @return Class of the default State Store driver implementation.
* @return Class of the State Store driver implementation.
*/
public static Class<? extends StateStoreDriver> getDefaultDriver() {
return DFSConfigKeys.FEDERATION_STORE_DRIVER_CLASS_DEFAULT;
public static Class<? extends StateStoreDriver> getTestDriverClass() {
return FEDERATION_STORE_DRIVER_CLASS_FOR_TEST;
}
/**
@ -69,7 +74,7 @@ public final class FederationStateStoreTestUtils {
* @return State Store configuration.
*/
public static Configuration getStateStoreConfiguration() {
Class<? extends StateStoreDriver> clazz = getDefaultDriver();
Class<? extends StateStoreDriver> clazz = getTestDriverClass();
return getStateStoreConfiguration(clazz);
}
@ -146,7 +151,7 @@ public final class FederationStateStoreTestUtils {
* @throws IOException
*/
public static void deleteStateStore() throws IOException {
Class<? extends StateStoreDriver> driverClass = getDefaultDriver();
Class<? extends StateStoreDriver> driverClass = getTestDriverClass();
deleteStateStore(driverClass);
}