Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-16605. Improve Code With Lambda in hadoop-hdfs-rbf moudle. (#4375)

This commit is contained in:
slfan1989 2022-07-22 09:42:55 +08:00 committed by GitHub
parent 08a940d5dd
commit 2f6916a313
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 100 additions and 157 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/PeriodicService.java
View File

@ -167,21 +167,18 @@ public abstract class PeriodicService extends AbstractService {
stopPeriodic();
// Create the runnable service
Runnable updateRunnable = new Runnable() {
@Override
public void run() {
LOG.debug("Running {} update task", serviceName);
try {
if (!isRunning) {
return;
}
periodicInvoke();
runCount++;
lastRun = Time.now();
} catch (Exception ex) {
errorCount++;
LOG.warn(serviceName + " service threw an exception", ex);
Runnable updateRunnable = () -> {
LOG.debug("Running {} update task", serviceName);
try {
if (!isRunning) {
return;
}
periodicInvoke();
runCount++;
lastRun = Time.now();
} catch (Exception ex) {
errorCount++;
LOG.warn("{} service threw an exception", serviceName, ex);
}
};

7
hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterHeartbeatService.java
View File

@ -63,12 +63,7 @@ public class RouterHeartbeatService extends PeriodicService {
* Trigger the update of the Router state asynchronously.
*/
protected void updateStateAsync() {
Thread thread = new Thread(new Runnable() {
@Override
public void run() {
updateStateStore();
}
}, "Router Heartbeat Async");
Thread thread = new Thread(this::updateStateStore, "Router Heartbeat Async");
thread.setDaemon(true);
thread.start();
}

2
hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
View File

@ -405,7 +405,7 @@ public class RouterRpcServer extends AbstractService implements ClientProtocol,
.asMap()
.keySet()
.parallelStream()
.forEach((key) -> this.dnCache.refresh(key)),
.forEach(this.dnCache::refresh),
0,
dnCacheExpire, TimeUnit.MILLISECONDS);
initRouterFedRename();

84
hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/FederationTestUtils.java
View File

@ -92,8 +92,6 @@ import org.mockito.stubbing.Answer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.function.Supplier;
/**
* Helper utilities for testing HDFS Federation.
*/
@ -174,26 +172,23 @@ public final class FederationTestUtils {
final String nsId, final String nnId,
final FederationNamenodeServiceState state) throws Exception {
GenericTestUtils.waitFor(new Supplier<Boolean>() {
@Override
public Boolean get() {
try {
List<? extends FederationNamenodeContext> namenodes =
resolver.getNamenodesForNameserviceId(nsId);
if (namenodes != null) {
for (FederationNamenodeContext namenode : namenodes) {
// Check if this is the Namenode we are checking
if (namenode.getNamenodeId() == nnId ||
namenode.getNamenodeId().equals(nnId)) {
return state == null || namenode.getState().equals(state);
}
GenericTestUtils.waitFor(() -> {
try {
List<? extends FederationNamenodeContext> namenodes =
resolver.getNamenodesForNameserviceId(nsId);
if (namenodes != null) {
for (FederationNamenodeContext namenode : namenodes) {
// Check if this is the Namenode we are checking
if (namenode.getNamenodeId() == nnId ||
namenode.getNamenodeId().equals(nnId)) {
return state == null || namenode.getState().equals(state);
}
}
} catch (IOException e) {
// Ignore
}
return false;
} catch (IOException e) {
// Ignore
}
return false;
}, 1000, 60 * 1000);
}
@ -209,22 +204,19 @@ public final class FederationTestUtils {
final ActiveNamenodeResolver resolver, final String nsId,
final FederationNamenodeServiceState state) throws Exception {
GenericTestUtils.waitFor(new Supplier<Boolean>() {
@Override
public Boolean get() {
try {
List<? extends FederationNamenodeContext> nns =
resolver.getNamenodesForNameserviceId(nsId);
for (FederationNamenodeContext nn : nns) {
if (nn.getState().equals(state)) {
return true;
}
GenericTestUtils.waitFor(() -> {
try {
List<? extends FederationNamenodeContext> nns =
resolver.getNamenodesForNameserviceId(nsId);
for (FederationNamenodeContext nn : nns) {
if (nn.getState().equals(state)) {
return true;
}
} catch (IOException e) {
// Ignore
}
return false;
} catch (IOException e) {
// Ignore
}
return false;
}, 1000, 20 * 1000);
}
@ -361,19 +353,16 @@ public final class FederationTestUtils {
*/
public static void waitRouterRegistered(RouterStore stateManager,
long routerCount, int timeout) throws Exception {
GenericTestUtils.waitFor(new Supplier<Boolean>() {
@Override
public Boolean get() {
try {
List<RouterState> cachedRecords = stateManager.getCachedRecords();
if (cachedRecords.size() == routerCount) {
return true;
}
} catch (IOException e) {
// Ignore
GenericTestUtils.waitFor(() -> {
try {
List<RouterState> cachedRecords = stateManager.getCachedRecords();
if (cachedRecords.size() == routerCount) {
return true;
}
return false;
} catch (IOException e) {
// Ignore
}
return false;
}, 100, timeout);
}
@ -390,13 +379,10 @@ public final class FederationTestUtils {
ConnectionManager connectionManager =
new ConnectionManager(server.getConfig());
ConnectionManager spyConnectionManager = spy(connectionManager);
doAnswer(new Answer() {
@Override
public Object answer(InvocationOnMock invocation) throws Throwable {
LOG.info("Simulating connectionManager throw IOException {}",
invocation.getMock());
throw new IOException("Simulate connectionManager throw IOException");
}
doAnswer(invocation -> {
LOG.info("Simulating connectionManager throw IOException {}",
invocation.getMock());
throw new IOException("Simulate connectionManager throw IOException");
}).when(spyConnectionManager).getConnection(
any(UserGroupInformation.class), any(String.class), any(Class.class));

16
hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MiniRouterDFSCluster.java
View File

@ -237,12 +237,8 @@ public class MiniRouterDFSCluster {
throws IOException, URISyntaxException, InterruptedException {
LOG.info("Connecting to router at {}", fileSystemUri);
return user.doAs(new PrivilegedExceptionAction<DFSClient>() {
@Override
public DFSClient run() throws IOException {
return new DFSClient(fileSystemUri, conf);
}
});
return user.doAs((PrivilegedExceptionAction<DFSClient>)
() -> new DFSClient(fileSystemUri, conf));
}
public RouterClient getAdminClient() throws IOException {
@ -384,12 +380,8 @@ public class MiniRouterDFSCluster {
throws IOException, URISyntaxException, InterruptedException {
LOG.info("Connecting to namenode at {}", fileSystemUri);
return user.doAs(new PrivilegedExceptionAction<DFSClient>() {
@Override
public DFSClient run() throws IOException {
return new DFSClient(fileSystemUri, conf);
}
});
return user.doAs((PrivilegedExceptionAction<DFSClient>)
() -> new DFSClient(fileSystemUri, conf));
}
public DFSClient getClient() throws IOException, URISyntaxException {

15
hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MockNamenode.java
View File

@ -156,15 +156,12 @@ public class MockNamenode {
NamespaceInfo nsInfo = new NamespaceInfo(1, this.nsId, this.nsId, 1);
when(mockNn.versionRequest()).thenReturn(nsInfo);
when(mockNn.getServiceStatus()).thenAnswer(new Answer<HAServiceStatus>() {
@Override
public HAServiceStatus answer(InvocationOnMock invocation)
throws Throwable {
HAServiceStatus haStatus = new HAServiceStatus(getHAServiceState());
haStatus.setNotReadyToBecomeActive("");
return haStatus;
}
});
when(mockNn.getServiceStatus()).
thenAnswer((Answer<HAServiceStatus>) invocation -> {
HAServiceStatus haStatus = new HAServiceStatus(getHAServiceState());
haStatus.setNotReadyToBecomeActive("");
return haStatus;
});
}
/**

10
hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/fairness/TestRouterRefreshFairnessPolicyController.java
View File

@ -133,9 +133,8 @@ public class TestRouterRefreshFairnessPolicyController {
// Spawn 100 concurrent refresh requests
Thread[] threads = new Thread[100];
for (int i = 0; i < 100; i++) {
threads[i] = new Thread(() -> {
client.refreshFairnessPolicyController(routerContext.getConf());
});
threads[i] = new Thread(() ->
client.refreshFairnessPolicyController(routerContext.getConf()));
}
for (Thread thread : threads) {
@ -182,9 +181,8 @@ public class TestRouterRefreshFairnessPolicyController {
final int newNs1Permits = 4;
conf.setInt(DFS_ROUTER_FAIR_HANDLER_COUNT_KEY_PREFIX + "ns0", newNs0Permits);
conf.setInt(DFS_ROUTER_FAIR_HANDLER_COUNT_KEY_PREFIX + "ns1", newNs1Permits);
Thread threadRefreshController = new Thread(() -> {
client.refreshFairnessPolicyController(routerContext.getConf());
});
Thread threadRefreshController = new Thread(() -> client.
refreshFairnessPolicyController(routerContext.getConf()));
threadRefreshController.start();
threadRefreshController.join();

9
hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/order/TestLocalResolver.java
View File

@ -43,7 +43,6 @@ import org.apache.hadoop.hdfs.server.federation.store.protocol.GetNamenodeRegist
import org.apache.hadoop.hdfs.server.federation.store.records.MembershipState;
import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
import org.junit.Test;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
/**
@ -78,12 +77,8 @@ public class TestLocalResolver {
StringBuilder sb = new StringBuilder("clientX");
LocalResolver localResolver = new LocalResolver(conf, router);
LocalResolver spyLocalResolver = spy(localResolver);
doAnswer(new Answer<String>() {
@Override
public String answer(InvocationOnMock invocation) throws Throwable {
return sb.toString();
}
}).when(spyLocalResolver).getClientAddr();
doAnswer((Answer<String>) invocation -> sb.toString()
).when(spyLocalResolver).getClientAddr();
// Add the mocks to the resolver
MultipleDestinationMountTableResolver resolver =

2
hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestConnectionManager.java
View File

@ -144,7 +144,7 @@ public class TestConnectionManager {
connectionCreator.setDaemon(true);
connectionCreator.start();
// Wait to make sure async thread is scheduled and picks
GenericTestUtils.waitFor(()->queue.isEmpty(), 50, 5000);
GenericTestUtils.waitFor(queue::isEmpty, 50, 5000);
// At this point connection creation task should be definitely picked up.
assertTrue(queue.isEmpty());
// At this point connection thread should still be alive.

50
hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterClientRejectOverload.java
View File

@ -206,33 +206,29 @@ public class TestRouterClientRejectOverload {
for (int i = 0; i < numOps; i++) {
// Stagger the operations a little (50ms)
final int sleepTime = i * 50;
Future<?> future = exec.submit(new Runnable() {
@Override
public void run() {
DFSClient routerClient = null;
try {
Thread.sleep(sleepTime);
routerClient = new DFSClient(address, conf);
String clientName = routerClient.getClientName();
ClientProtocol routerProto = routerClient.getNamenode();
routerProto.renewLease(clientName, null);
} catch (RemoteException re) {
IOException ioe = re.unwrapRemoteException();
assertTrue("Wrong exception: " + ioe,
ioe instanceof StandbyException);
assertExceptionContains("is overloaded", ioe);
overloadException.incrementAndGet();
} catch (IOException e) {
fail("Unexpected exception: " + e);
} catch (InterruptedException e) {
fail("Cannot sleep: " + e);
} finally {
if (routerClient != null) {
try {
routerClient.close();
} catch (IOException e) {
LOG.error("Cannot close the client");
}
Future<?> future = exec.submit(() -> {
DFSClient routerClient = null;
try {
Thread.sleep(sleepTime);
routerClient = new DFSClient(address, conf);
String clientName = routerClient.getClientName();
ClientProtocol routerProto = routerClient.getNamenode();
routerProto.renewLease(clientName, null);
} catch (RemoteException re) {
IOException ioe = re.unwrapRemoteException();
assertTrue("Wrong exception: " + ioe, ioe instanceof StandbyException);
assertExceptionContains("is overloaded", ioe);
overloadException.incrementAndGet();
} catch (IOException e) {
fail("Unexpected exception: " + e);
} catch (InterruptedException e) {
fail("Cannot sleep: " + e);
} finally {
if (routerClient != null) {
try {
routerClient.close();
} catch (IOException e) {
LOG.error("Cannot close the client");
}
}
}

2
hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterFaultTolerant.java
View File

@ -168,7 +168,7 @@ public class TestRouterFaultTolerant {
}
namenodes.clear();
routers.forEach(router -> router.stop());
routers.forEach(Router::stop);
routers.clear();
if (service != null) {

26
hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuota.java
View File

@ -74,8 +74,6 @@ import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import java.util.function.Supplier;
/**
* Tests quota behaviors in Router-based Federation.
*/
@ -210,21 +208,17 @@ public class TestRouterQuota {
routerClient.create("/ssquota/file", true).close();
routerClient.create("/ssquota/subdir/file", true).close();
GenericTestUtils.waitFor(new Supplier<Boolean>() {
@Override
public Boolean get() {
boolean isDsQuotaViolated = false;
try {
// append data to trigger NSQuotaExceededException
appendData("/ssquota/file", routerClient, BLOCK_SIZE);
appendData("/ssquota/subdir/file", routerClient, BLOCK_SIZE);
} catch (DSQuotaExceededException e) {
isDsQuotaViolated = true;
} catch (IOException ignored) {
}
return isDsQuotaViolated;
GenericTestUtils.waitFor(() -> {
boolean isDsQuotaViolated = false;
try {
// append data to trigger NSQuotaExceededException
appendData("/ssquota/file", routerClient, BLOCK_SIZE);
appendData("/ssquota/subdir/file", routerClient, BLOCK_SIZE);
} catch (DSQuotaExceededException e) {
isDsQuotaViolated = true;
} catch (IOException ignored) {
}
return isDsQuotaViolated;
}, 5000, 60000);
// append data to destination path in real FileSystem should be okay

9
hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRPCClientRetries.java
View File

@ -54,8 +54,6 @@ import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.Timeout;
import java.util.function.Supplier;
/**
* Test retry behavior of the Router RPC Client.
*/
@ -237,11 +235,6 @@ public class TestRouterRPCClientRetries {
private static void waitUpdateLiveNodes(
final String oldValue, final NamenodeBeanMetrics metrics)
throws Exception {
waitFor(new Supplier<Boolean>() {
@Override
public Boolean get() {
return !oldValue.equals(metrics.getLiveNodes());
}
}, 500, 5 * 1000);
waitFor(() -> !oldValue.equals(metrics.getLiveNodes()), 500, 5 * 1000);
}
}