HDFS-16605. Improve Code With Lambda in hadoop-hdfs-rbf moudle. (#4375)
This commit is contained in:
parent
08a940d5dd
commit
2f6916a313
|
@ -167,9 +167,7 @@ public abstract class PeriodicService extends AbstractService {
|
||||||
stopPeriodic();
|
stopPeriodic();
|
||||||
|
|
||||||
// Create the runnable service
|
// Create the runnable service
|
||||||
Runnable updateRunnable = new Runnable() {
|
Runnable updateRunnable = () -> {
|
||||||
@Override
|
|
||||||
public void run() {
|
|
||||||
LOG.debug("Running {} update task", serviceName);
|
LOG.debug("Running {} update task", serviceName);
|
||||||
try {
|
try {
|
||||||
if (!isRunning) {
|
if (!isRunning) {
|
||||||
|
@ -180,8 +178,7 @@ public abstract class PeriodicService extends AbstractService {
|
||||||
lastRun = Time.now();
|
lastRun = Time.now();
|
||||||
} catch (Exception ex) {
|
} catch (Exception ex) {
|
||||||
errorCount++;
|
errorCount++;
|
||||||
LOG.warn(serviceName + " service threw an exception", ex);
|
LOG.warn("{} service threw an exception", serviceName, ex);
|
||||||
}
|
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
|
@ -63,12 +63,7 @@ public class RouterHeartbeatService extends PeriodicService {
|
||||||
* Trigger the update of the Router state asynchronously.
|
* Trigger the update of the Router state asynchronously.
|
||||||
*/
|
*/
|
||||||
protected void updateStateAsync() {
|
protected void updateStateAsync() {
|
||||||
Thread thread = new Thread(new Runnable() {
|
Thread thread = new Thread(this::updateStateStore, "Router Heartbeat Async");
|
||||||
@Override
|
|
||||||
public void run() {
|
|
||||||
updateStateStore();
|
|
||||||
}
|
|
||||||
}, "Router Heartbeat Async");
|
|
||||||
thread.setDaemon(true);
|
thread.setDaemon(true);
|
||||||
thread.start();
|
thread.start();
|
||||||
}
|
}
|
||||||
|
|
|
@ -405,7 +405,7 @@ public class RouterRpcServer extends AbstractService implements ClientProtocol,
|
||||||
.asMap()
|
.asMap()
|
||||||
.keySet()
|
.keySet()
|
||||||
.parallelStream()
|
.parallelStream()
|
||||||
.forEach((key) -> this.dnCache.refresh(key)),
|
.forEach(this.dnCache::refresh),
|
||||||
0,
|
0,
|
||||||
dnCacheExpire, TimeUnit.MILLISECONDS);
|
dnCacheExpire, TimeUnit.MILLISECONDS);
|
||||||
initRouterFedRename();
|
initRouterFedRename();
|
||||||
|
|
|
@ -92,8 +92,6 @@ import org.mockito.stubbing.Answer;
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
import java.util.function.Supplier;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Helper utilities for testing HDFS Federation.
|
* Helper utilities for testing HDFS Federation.
|
||||||
*/
|
*/
|
||||||
|
@ -174,9 +172,7 @@ public final class FederationTestUtils {
|
||||||
final String nsId, final String nnId,
|
final String nsId, final String nnId,
|
||||||
final FederationNamenodeServiceState state) throws Exception {
|
final FederationNamenodeServiceState state) throws Exception {
|
||||||
|
|
||||||
GenericTestUtils.waitFor(new Supplier<Boolean>() {
|
GenericTestUtils.waitFor(() -> {
|
||||||
@Override
|
|
||||||
public Boolean get() {
|
|
||||||
try {
|
try {
|
||||||
List<? extends FederationNamenodeContext> namenodes =
|
List<? extends FederationNamenodeContext> namenodes =
|
||||||
resolver.getNamenodesForNameserviceId(nsId);
|
resolver.getNamenodesForNameserviceId(nsId);
|
||||||
|
@ -193,7 +189,6 @@ public final class FederationTestUtils {
|
||||||
// Ignore
|
// Ignore
|
||||||
}
|
}
|
||||||
return false;
|
return false;
|
||||||
}
|
|
||||||
}, 1000, 60 * 1000);
|
}, 1000, 60 * 1000);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -209,9 +204,7 @@ public final class FederationTestUtils {
|
||||||
final ActiveNamenodeResolver resolver, final String nsId,
|
final ActiveNamenodeResolver resolver, final String nsId,
|
||||||
final FederationNamenodeServiceState state) throws Exception {
|
final FederationNamenodeServiceState state) throws Exception {
|
||||||
|
|
||||||
GenericTestUtils.waitFor(new Supplier<Boolean>() {
|
GenericTestUtils.waitFor(() -> {
|
||||||
@Override
|
|
||||||
public Boolean get() {
|
|
||||||
try {
|
try {
|
||||||
List<? extends FederationNamenodeContext> nns =
|
List<? extends FederationNamenodeContext> nns =
|
||||||
resolver.getNamenodesForNameserviceId(nsId);
|
resolver.getNamenodesForNameserviceId(nsId);
|
||||||
|
@ -224,7 +217,6 @@ public final class FederationTestUtils {
|
||||||
// Ignore
|
// Ignore
|
||||||
}
|
}
|
||||||
return false;
|
return false;
|
||||||
}
|
|
||||||
}, 1000, 20 * 1000);
|
}, 1000, 20 * 1000);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -361,9 +353,7 @@ public final class FederationTestUtils {
|
||||||
*/
|
*/
|
||||||
public static void waitRouterRegistered(RouterStore stateManager,
|
public static void waitRouterRegistered(RouterStore stateManager,
|
||||||
long routerCount, int timeout) throws Exception {
|
long routerCount, int timeout) throws Exception {
|
||||||
GenericTestUtils.waitFor(new Supplier<Boolean>() {
|
GenericTestUtils.waitFor(() -> {
|
||||||
@Override
|
|
||||||
public Boolean get() {
|
|
||||||
try {
|
try {
|
||||||
List<RouterState> cachedRecords = stateManager.getCachedRecords();
|
List<RouterState> cachedRecords = stateManager.getCachedRecords();
|
||||||
if (cachedRecords.size() == routerCount) {
|
if (cachedRecords.size() == routerCount) {
|
||||||
|
@ -373,7 +363,6 @@ public final class FederationTestUtils {
|
||||||
// Ignore
|
// Ignore
|
||||||
}
|
}
|
||||||
return false;
|
return false;
|
||||||
}
|
|
||||||
}, 100, timeout);
|
}, 100, timeout);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -390,13 +379,10 @@ public final class FederationTestUtils {
|
||||||
ConnectionManager connectionManager =
|
ConnectionManager connectionManager =
|
||||||
new ConnectionManager(server.getConfig());
|
new ConnectionManager(server.getConfig());
|
||||||
ConnectionManager spyConnectionManager = spy(connectionManager);
|
ConnectionManager spyConnectionManager = spy(connectionManager);
|
||||||
doAnswer(new Answer() {
|
doAnswer(invocation -> {
|
||||||
@Override
|
|
||||||
public Object answer(InvocationOnMock invocation) throws Throwable {
|
|
||||||
LOG.info("Simulating connectionManager throw IOException {}",
|
LOG.info("Simulating connectionManager throw IOException {}",
|
||||||
invocation.getMock());
|
invocation.getMock());
|
||||||
throw new IOException("Simulate connectionManager throw IOException");
|
throw new IOException("Simulate connectionManager throw IOException");
|
||||||
}
|
|
||||||
}).when(spyConnectionManager).getConnection(
|
}).when(spyConnectionManager).getConnection(
|
||||||
any(UserGroupInformation.class), any(String.class), any(Class.class));
|
any(UserGroupInformation.class), any(String.class), any(Class.class));
|
||||||
|
|
||||||
|
|
|
@ -237,12 +237,8 @@ public class MiniRouterDFSCluster {
|
||||||
throws IOException, URISyntaxException, InterruptedException {
|
throws IOException, URISyntaxException, InterruptedException {
|
||||||
|
|
||||||
LOG.info("Connecting to router at {}", fileSystemUri);
|
LOG.info("Connecting to router at {}", fileSystemUri);
|
||||||
return user.doAs(new PrivilegedExceptionAction<DFSClient>() {
|
return user.doAs((PrivilegedExceptionAction<DFSClient>)
|
||||||
@Override
|
() -> new DFSClient(fileSystemUri, conf));
|
||||||
public DFSClient run() throws IOException {
|
|
||||||
return new DFSClient(fileSystemUri, conf);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public RouterClient getAdminClient() throws IOException {
|
public RouterClient getAdminClient() throws IOException {
|
||||||
|
@ -384,12 +380,8 @@ public class MiniRouterDFSCluster {
|
||||||
throws IOException, URISyntaxException, InterruptedException {
|
throws IOException, URISyntaxException, InterruptedException {
|
||||||
|
|
||||||
LOG.info("Connecting to namenode at {}", fileSystemUri);
|
LOG.info("Connecting to namenode at {}", fileSystemUri);
|
||||||
return user.doAs(new PrivilegedExceptionAction<DFSClient>() {
|
return user.doAs((PrivilegedExceptionAction<DFSClient>)
|
||||||
@Override
|
() -> new DFSClient(fileSystemUri, conf));
|
||||||
public DFSClient run() throws IOException {
|
|
||||||
return new DFSClient(fileSystemUri, conf);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public DFSClient getClient() throws IOException, URISyntaxException {
|
public DFSClient getClient() throws IOException, URISyntaxException {
|
||||||
|
|
|
@ -156,14 +156,11 @@ public class MockNamenode {
|
||||||
NamespaceInfo nsInfo = new NamespaceInfo(1, this.nsId, this.nsId, 1);
|
NamespaceInfo nsInfo = new NamespaceInfo(1, this.nsId, this.nsId, 1);
|
||||||
when(mockNn.versionRequest()).thenReturn(nsInfo);
|
when(mockNn.versionRequest()).thenReturn(nsInfo);
|
||||||
|
|
||||||
when(mockNn.getServiceStatus()).thenAnswer(new Answer<HAServiceStatus>() {
|
when(mockNn.getServiceStatus()).
|
||||||
@Override
|
thenAnswer((Answer<HAServiceStatus>) invocation -> {
|
||||||
public HAServiceStatus answer(InvocationOnMock invocation)
|
|
||||||
throws Throwable {
|
|
||||||
HAServiceStatus haStatus = new HAServiceStatus(getHAServiceState());
|
HAServiceStatus haStatus = new HAServiceStatus(getHAServiceState());
|
||||||
haStatus.setNotReadyToBecomeActive("");
|
haStatus.setNotReadyToBecomeActive("");
|
||||||
return haStatus;
|
return haStatus;
|
||||||
}
|
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -133,9 +133,8 @@ public class TestRouterRefreshFairnessPolicyController {
|
||||||
// Spawn 100 concurrent refresh requests
|
// Spawn 100 concurrent refresh requests
|
||||||
Thread[] threads = new Thread[100];
|
Thread[] threads = new Thread[100];
|
||||||
for (int i = 0; i < 100; i++) {
|
for (int i = 0; i < 100; i++) {
|
||||||
threads[i] = new Thread(() -> {
|
threads[i] = new Thread(() ->
|
||||||
client.refreshFairnessPolicyController(routerContext.getConf());
|
client.refreshFairnessPolicyController(routerContext.getConf()));
|
||||||
});
|
|
||||||
}
|
}
|
||||||
|
|
||||||
for (Thread thread : threads) {
|
for (Thread thread : threads) {
|
||||||
|
@ -182,9 +181,8 @@ public class TestRouterRefreshFairnessPolicyController {
|
||||||
final int newNs1Permits = 4;
|
final int newNs1Permits = 4;
|
||||||
conf.setInt(DFS_ROUTER_FAIR_HANDLER_COUNT_KEY_PREFIX + "ns0", newNs0Permits);
|
conf.setInt(DFS_ROUTER_FAIR_HANDLER_COUNT_KEY_PREFIX + "ns0", newNs0Permits);
|
||||||
conf.setInt(DFS_ROUTER_FAIR_HANDLER_COUNT_KEY_PREFIX + "ns1", newNs1Permits);
|
conf.setInt(DFS_ROUTER_FAIR_HANDLER_COUNT_KEY_PREFIX + "ns1", newNs1Permits);
|
||||||
Thread threadRefreshController = new Thread(() -> {
|
Thread threadRefreshController = new Thread(() -> client.
|
||||||
client.refreshFairnessPolicyController(routerContext.getConf());
|
refreshFairnessPolicyController(routerContext.getConf()));
|
||||||
});
|
|
||||||
threadRefreshController.start();
|
threadRefreshController.start();
|
||||||
threadRefreshController.join();
|
threadRefreshController.join();
|
||||||
|
|
||||||
|
|
|
@ -43,7 +43,6 @@ import org.apache.hadoop.hdfs.server.federation.store.protocol.GetNamenodeRegist
|
||||||
import org.apache.hadoop.hdfs.server.federation.store.records.MembershipState;
|
import org.apache.hadoop.hdfs.server.federation.store.records.MembershipState;
|
||||||
import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
|
import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
import org.mockito.invocation.InvocationOnMock;
|
|
||||||
import org.mockito.stubbing.Answer;
|
import org.mockito.stubbing.Answer;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -78,12 +77,8 @@ public class TestLocalResolver {
|
||||||
StringBuilder sb = new StringBuilder("clientX");
|
StringBuilder sb = new StringBuilder("clientX");
|
||||||
LocalResolver localResolver = new LocalResolver(conf, router);
|
LocalResolver localResolver = new LocalResolver(conf, router);
|
||||||
LocalResolver spyLocalResolver = spy(localResolver);
|
LocalResolver spyLocalResolver = spy(localResolver);
|
||||||
doAnswer(new Answer<String>() {
|
doAnswer((Answer<String>) invocation -> sb.toString()
|
||||||
@Override
|
).when(spyLocalResolver).getClientAddr();
|
||||||
public String answer(InvocationOnMock invocation) throws Throwable {
|
|
||||||
return sb.toString();
|
|
||||||
}
|
|
||||||
}).when(spyLocalResolver).getClientAddr();
|
|
||||||
|
|
||||||
// Add the mocks to the resolver
|
// Add the mocks to the resolver
|
||||||
MultipleDestinationMountTableResolver resolver =
|
MultipleDestinationMountTableResolver resolver =
|
||||||
|
|
|
@ -144,7 +144,7 @@ public class TestConnectionManager {
|
||||||
connectionCreator.setDaemon(true);
|
connectionCreator.setDaemon(true);
|
||||||
connectionCreator.start();
|
connectionCreator.start();
|
||||||
// Wait to make sure async thread is scheduled and picks
|
// Wait to make sure async thread is scheduled and picks
|
||||||
GenericTestUtils.waitFor(()->queue.isEmpty(), 50, 5000);
|
GenericTestUtils.waitFor(queue::isEmpty, 50, 5000);
|
||||||
// At this point connection creation task should be definitely picked up.
|
// At this point connection creation task should be definitely picked up.
|
||||||
assertTrue(queue.isEmpty());
|
assertTrue(queue.isEmpty());
|
||||||
// At this point connection thread should still be alive.
|
// At this point connection thread should still be alive.
|
||||||
|
|
|
@ -206,9 +206,7 @@ public class TestRouterClientRejectOverload {
|
||||||
for (int i = 0; i < numOps; i++) {
|
for (int i = 0; i < numOps; i++) {
|
||||||
// Stagger the operations a little (50ms)
|
// Stagger the operations a little (50ms)
|
||||||
final int sleepTime = i * 50;
|
final int sleepTime = i * 50;
|
||||||
Future<?> future = exec.submit(new Runnable() {
|
Future<?> future = exec.submit(() -> {
|
||||||
@Override
|
|
||||||
public void run() {
|
|
||||||
DFSClient routerClient = null;
|
DFSClient routerClient = null;
|
||||||
try {
|
try {
|
||||||
Thread.sleep(sleepTime);
|
Thread.sleep(sleepTime);
|
||||||
|
@ -218,8 +216,7 @@ public class TestRouterClientRejectOverload {
|
||||||
routerProto.renewLease(clientName, null);
|
routerProto.renewLease(clientName, null);
|
||||||
} catch (RemoteException re) {
|
} catch (RemoteException re) {
|
||||||
IOException ioe = re.unwrapRemoteException();
|
IOException ioe = re.unwrapRemoteException();
|
||||||
assertTrue("Wrong exception: " + ioe,
|
assertTrue("Wrong exception: " + ioe, ioe instanceof StandbyException);
|
||||||
ioe instanceof StandbyException);
|
|
||||||
assertExceptionContains("is overloaded", ioe);
|
assertExceptionContains("is overloaded", ioe);
|
||||||
overloadException.incrementAndGet();
|
overloadException.incrementAndGet();
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
|
@ -235,7 +232,6 @@ public class TestRouterClientRejectOverload {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
});
|
});
|
||||||
futures.add(future);
|
futures.add(future);
|
||||||
}
|
}
|
||||||
|
|
|
@ -168,7 +168,7 @@ public class TestRouterFaultTolerant {
|
||||||
}
|
}
|
||||||
namenodes.clear();
|
namenodes.clear();
|
||||||
|
|
||||||
routers.forEach(router -> router.stop());
|
routers.forEach(Router::stop);
|
||||||
routers.clear();
|
routers.clear();
|
||||||
|
|
||||||
if (service != null) {
|
if (service != null) {
|
||||||
|
|
|
@ -74,8 +74,6 @@ import org.junit.After;
|
||||||
import org.junit.Before;
|
import org.junit.Before;
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
|
|
||||||
import java.util.function.Supplier;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Tests quota behaviors in Router-based Federation.
|
* Tests quota behaviors in Router-based Federation.
|
||||||
*/
|
*/
|
||||||
|
@ -210,10 +208,7 @@ public class TestRouterQuota {
|
||||||
routerClient.create("/ssquota/file", true).close();
|
routerClient.create("/ssquota/file", true).close();
|
||||||
routerClient.create("/ssquota/subdir/file", true).close();
|
routerClient.create("/ssquota/subdir/file", true).close();
|
||||||
|
|
||||||
GenericTestUtils.waitFor(new Supplier<Boolean>() {
|
GenericTestUtils.waitFor(() -> {
|
||||||
|
|
||||||
@Override
|
|
||||||
public Boolean get() {
|
|
||||||
boolean isDsQuotaViolated = false;
|
boolean isDsQuotaViolated = false;
|
||||||
try {
|
try {
|
||||||
// append data to trigger NSQuotaExceededException
|
// append data to trigger NSQuotaExceededException
|
||||||
|
@ -224,7 +219,6 @@ public class TestRouterQuota {
|
||||||
} catch (IOException ignored) {
|
} catch (IOException ignored) {
|
||||||
}
|
}
|
||||||
return isDsQuotaViolated;
|
return isDsQuotaViolated;
|
||||||
}
|
|
||||||
}, 5000, 60000);
|
}, 5000, 60000);
|
||||||
|
|
||||||
// append data to destination path in real FileSystem should be okay
|
// append data to destination path in real FileSystem should be okay
|
||||||
|
|
|
@ -54,8 +54,6 @@ import org.junit.Rule;
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
import org.junit.rules.Timeout;
|
import org.junit.rules.Timeout;
|
||||||
|
|
||||||
import java.util.function.Supplier;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Test retry behavior of the Router RPC Client.
|
* Test retry behavior of the Router RPC Client.
|
||||||
*/
|
*/
|
||||||
|
@ -237,11 +235,6 @@ public class TestRouterRPCClientRetries {
|
||||||
private static void waitUpdateLiveNodes(
|
private static void waitUpdateLiveNodes(
|
||||||
final String oldValue, final NamenodeBeanMetrics metrics)
|
final String oldValue, final NamenodeBeanMetrics metrics)
|
||||||
throws Exception {
|
throws Exception {
|
||||||
waitFor(new Supplier<Boolean>() {
|
waitFor(() -> !oldValue.equals(metrics.getLiveNodes()), 500, 5 * 1000);
|
||||||
@Override
|
|
||||||
public Boolean get() {
|
|
||||||
return !oldValue.equals(metrics.getLiveNodes());
|
|
||||||
}
|
|
||||||
}, 500, 5 * 1000);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue